diff --git a/.gitmodules b/.gitmodules index 51d8eac03..6ebcea592 100644 --- a/.gitmodules +++ b/.gitmodules @@ -6,3 +6,6 @@ path = text_to_image/torchtitan url = https://github.com/pytorch/torchtitan.git branch = mlperf-training-flux.1 +[submodule "recommendation_v4/cutlass"] + path = recommendation_v4/generative_recommenders/ops/cpp/cutlass + url = https://github.com/NVIDIA/cutlass.git diff --git a/recommendation_v4/.gitignore b/recommendation_v4/.gitignore new file mode 100644 index 000000000..5edddc5b3 --- /dev/null +++ b/recommendation_v4/.gitignore @@ -0,0 +1,159 @@ +# Don't check in parsed data files and other temporary files +tmp/ +exps/ +ckpts/ +results/ + +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version + +# pipenv +# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. +# However, in case of collaboration, if having platform-specific dependencies or dependencies +# having no cross-platform support, pipenv may install dependencies that don't work, or not +# install all needed dependencies. +#Pipfile.lock + +# poetry +# Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. +# This is especially recommended for binary packages to ensure reproducibility, and is more +# commonly ignored for libraries. +# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control +#poetry.lock + +# pdm +# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. +#pdm.lock +# pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it +# in version control. +# https://pdm.fming.dev/#use-with-ide +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/recommendation_v4/Dockerfile b/recommendation_v4/Dockerfile new file mode 100644 index 000000000..450a5ab55 --- /dev/null +++ b/recommendation_v4/Dockerfile @@ -0,0 +1,86 @@ +# MI350X path — implements docs/training_recipe.md §"MI350X". + +FROM rocm/primus:v26.3 + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 \ + PIP_DISABLE_PIP_VERSION_CHECK=1 + +WORKDIR /workspace/recommendation_v4 + +# torch / torchvision / torchaudio — training_recipe.md:38-40. +RUN pip install --upgrade --no-deps \ + --index-url https://download.pytorch.org/whl/rocm7.2 \ + torch==2.12.0+rocm7.2 \ + torchvision==0.27.0+rocm7.2 \ + torchaudio==2.11.0+rocm7.2 + +# torchrec — training_recipe.md:43. +RUN pip install --force-reinstall --no-deps \ + "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00" + +# fbgemm_gpu — training_recipe.md:42. Build from FBGEMM commit 10b77573 for +# gfx950 against the replaced torch. ~30-60 min. +RUN apt-get update && apt-get install -y --no-install-recommends git build-essential && \ + rm -rf /var/lib/apt/lists/* && \ + git clone --recursive https://github.com/pytorch/FBGEMM.git /tmp/FBGEMM && \ + cd /tmp/FBGEMM && \ + git checkout 10b775730212923f65f7b78f79b6a01d80cf3c29 && \ + git submodule update --init --recursive && \ + cd fbgemm_gpu && \ + # Filter `fairscale` and the torch family from fbgemm's requirements.txt: + # fairscale pulls a CPU torch that would clobber the +rocm7.2 wheel installed + # above. fairscale is a distributed-training lib used by fbgemm tests, not + # by the build itself. + grep -v -E '^(fairscale|torch|torchvision|torchaudio)([<>=!]|$)' requirements.txt > /tmp/req.txt && \ + pip install -r /tmp/req.txt && \ + python setup.py -j 32 bdist_wheel \ + --build-target=default \ + --build-variant=rocm \ + -DHIP_ROOT_DIR=/opt/rocm \ + -DAMDGPU_TARGETS=gfx950 && \ + pip install --force-reinstall --no-deps dist/fbgemm_gpu_nightly_rocm*.whl && \ + cd / && rm -rf /tmp/FBGEMM + +# polars-u64-idx — training_recipe.md:44 (mandatory; yambda-5b > 4.29 B rows). +# Remaining packages — training_recipe.md:156-159 ("Additional Python deps") plus +# `datasets` + `huggingface_hub`, which the recipe does not list but +# preprocess_public_data.py:278 imports to download yambda from HuggingFace. +RUN pip install \ + polars-u64-idx==1.33.1 \ + gin-config \ + absl-py \ + datasets \ + huggingface_hub \ + pyre-extensions \ + iopath \ + typing-inspect \ + psutil \ + tqdm \ + pyyaml \ + lightning-utilities && \ + # torchmetrics and tensordict declare `torch` as a dep; without --no-deps + # pip pulls torch==2.12.0+cu130 from PyPI which clobbers the +rocm7.2 wheel + # we installed above (libtorch_hip.so disappears, fbgemm_gpu fails to load). + pip install --no-deps \ + torchmetrics==1.0.3 \ + tensordict + +# mlperf_logging — required by train/mlperf_logging_utils.py for MLPerf +# compliance logs. Pinned to the Training 6.0 tag for reproducibility; --no-deps +# so pip does not resolve requirements.txt's torch/fbgemm_gpu/torchrec pins and +# clobber the +rocm7.2 wheels above. +RUN pip install --no-deps "git+https://github.com/mlcommons/logging.git@6.0.0-rc6" + +# Smoke-test the 6 imports the launch script checks at +# scripts/launch_smoke_8gpu.sh:26. +RUN python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; \ +print('torch', torch.__version__, '| hip', getattr(torch.version, 'hip', None))" + +COPY . /workspace/recommendation_v4 + +ENV PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True \ + HSTU_HAMMER_KERNEL=TRITON \ + DLRM_DATA_PATH=/data/mlperf_dlrm_v4 + +CMD ["bash"] diff --git a/recommendation_v4/LICENSE b/recommendation_v4/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/recommendation_v4/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/recommendation_v4/README.MD b/recommendation_v4/README.MD new file mode 100644 index 000000000..fb6f65b7a --- /dev/null +++ b/recommendation_v4/README.MD @@ -0,0 +1,424 @@ +# Recommendation v4 — HSTU sequential recommendation (Yambda-5b) + +MLPerf Training reference benchmark. This is a fork of +[meta-recsys/generative-recommenders](https://github.com/meta-recsys/generative-recommenders) +extended to train an HSTU (Hierarchical Sequential Transduction Units) ranking +model on the [Yambda-5b](https://huggingface.co/datasets/yandex/yambda) +music-recommendation dataset, sized as an MLPerf-style training benchmark inside +the `mlcommons/training` tree. + +## 1. Summary + +This benchmark trains a model that predicts what a person will listen to next. +Given the history of songs a user has played, liked, or skipped, the model +learns to rank which song the user is most likely to genuinely listen to (rather +than skip) next. This is the same kind of "what should we recommend next?" +problem that powers music and video streaming feeds. The model is trained on a +large public dataset of anonymized music-listening events and is scored on how +well it predicts future listens it has never seen. + +## 2. Benchmark overview (technical) + +The model is a **sequential recommender**: instead of treating each interaction +independently (as classic click-through-rate models like DLRM-DCNv2 do), it +consumes a user's chronologically ordered interaction history as a sequence and +applies a Transformer-style attention stack (HSTU) over it. Each training +example is one "anchor" listen event together with that user's prior history +(user interaction history, or UIH) and a set of contextual/cross features. The +supervised target is a binary `listen_plus` label (a real listen: played for at +least 50% of the track) versus a skip. + +Training is **streaming / temporal-order**: the timeline is sliced into +fixed-duration windows and the model trains on window `T` then evaluates on the +strictly-future window `T+1`, so every reported metric is genuine +next-period generalization with no future leakage. The quality metric is +**AUC** on the held-out future window, and the convergence target is +**AUC >= 0.75**. + +The reference runs on 8 GPUs (validated on AMD Instinct MI350X / MI355X and +NVIDIA B200; see [docs/training_recipe.md](docs/training_recipe.md)) and scales +to multi-node via SLURM. + +## 3. Directions — steps to run + +The benchmark follows the standard MLPerf reference script flow: + +```bash +# 0. build/enter the container (canonical frozen environment) +docker build -t recommendation_v4 . +docker run --rm -it --device=/dev/kfd --device=/dev/dri \ + -v /path/to/dlrm_data:/data/mlperf_dlrm_v4 recommendation_v4 + +# 1. download + preprocess the dataset +DLRM_DATA_PATH=/data/mlperf_dlrm_v4 ./download_dataset.sh + +# 2. verify the preprocessed dataset +DLRM_DATA_PATH=/data/mlperf_dlrm_v4 ./verify_dataset.sh + +# 3. run the benchmark to the quality target and report wall-clock time +DLRM_DATA_PATH=/data/mlperf_dlrm_v4 ./run_and_time.sh +``` + +- [`download_dataset.sh`](download_dataset.sh) wraps the preprocessing pipeline + in `generative_recommenders.dlrm_v3.preprocess_public_data` (HuggingFace + download + temporal split + session segmentation + item-popularity counts). +- [`verify_dataset.sh`](verify_dataset.sh) checks the preprocessed files against + [`md5sums_yambda_5b_processed.txt`](md5sums_yambda_5b_processed.txt) (falls + back to a layout check until the canonical checksums are pinned). +- [`run_and_time.sh`](run_and_time.sh) runs the full-reference streaming + train+eval sweep on a single 8-GPU host with `AUC_THRESHOLD=0.75` and MLPerf + compliance logging, printing the elapsed time of the timed region. + +### 3.1 Multi-node (SLURM) + +For N >= 1 nodes use [`scripts/launch_slurm.sh`](scripts/launch_slurm.sh), which +provisions the container on each node and launches the same trainer. A bare +submit runs a small functional smoke run; set the run-shape knobs for the full +sweep: + +```bash +# smoke (fast functional check) +sbatch --nodes=1 scripts/launch_slurm.sh + +# full reference sweep +START_TS=0 NUM_TRAIN_TS=299 \ +NUM_TRAIN_BATCHES=0 NUM_EVAL_BATCHES=0 \ +EVAL_EVERY_N_WINDOWS=0 EVAL_EVERY_DATA_PCT=0.005 \ +AUC_THRESHOLD=0.75 \ +sbatch --nodes=2 scripts/launch_slurm.sh +``` + +Multi-node uses real RDMA (RoCEv2); the fabric/NCCL setup is documented in +[docs/multi_node_config.md](docs/multi_node_config.md). Keep all run outputs +(log, checkpoints, mllog, TensorBoard) under a writable scratch path you own — +the dataset mount is read-only. + +## 4. Model + +The model is **HSTU** (Hierarchical Sequential Transduction Units), the +generative-recommender architecture from Meta's ICML'24 paper *Actions Speak +Louder than Words: Trillion-Parameter Sequential Transducers for Generative +Recommendations* ([arXiv:2402.17152](https://arxiv.org/abs/2402.17152)). + +HSTU replaces the feature-interaction stack of a classic DLRM with a stack of +pointwise-attention "transducer" layers operating over the user's interaction +sequence. In this benchmark (the `dlrm_v3` path): + +- **Embeddings**: sparse tables for `item_id`, `artist_id`, `album_id`, `uid`, + and 7 cross-feature hashes (e.g. `user_x_artist`, `item_x_hour`), sharded + across GPUs with TorchRec `DistributedModelParallel`. +- **Sequence model**: an HSTU attention stack (`HSTU_NUM_LAYERS`, default 3) + over the interleaved UIH, computed with a fused jagged-attention Triton kernel + in bf16. +- **Supervision**: a single `listen_plus` binary task. The candidate event's + `action_weight` carries the supervision bit, and BCE loss is masked to + listen_plus candidates. + +See `generative_recommenders/dlrm_v3/configs.py` +(`get_hstu_configs`, `get_embedding_table_config`) for the exact architecture +and table specs, and the upstream README for the original modeling code. + +## 5. Dataset + +[Yambda-5b](https://huggingface.co/datasets/yandex/yambda) is a public +anonymized music-recommendation dataset from Yandex. The `5b` variant is used +for the reference. Statistics after preprocessing: + +| | | +|---|---| +| Total interaction events | **4.76 B** | +| Unique users | **1.00 M** | +| Max events per user | 27,738 | +| Median events per user | 2,695 | +| Mean events per user | 4,763 | +| Train events (300d) | 4.76 B | +| Test events (1d) | 22.4 M | +| Item catalog size | 9.39 M | + +### 5.1 Per-event-type distribution (across the full 4.76 B corpus) + +| Pool | Definition | Count | Share | +|---|---|---|---| +| **listen_plus (lp)** | `is_listen AND played_ratio >= 50%` | 2.92 B | **61.3%** | +| **skip** | `is_listen AND played_ratio < 50%` | 1.71 B | **35.9%** | +| **like** | explicit thumbs-up action | 89 M | **1.9%** | +| other | dislike / unlike / undislike | 47 M | 1.0% | + +The `like` pool is roughly **30x rarer** than `lp` — important context for the +gather strategy in §6. + +### 5.2 Preprocessing & download + +`./download_dataset.sh` (which calls +`python3 -m generative_recommenders.dlrm_v3.preprocess_public_data --dataset +yambda-5b --data-path `) downloads the 5b variant from HuggingFace, then: + +1. **Encodes** the raw `event_type` string into a uint8 lookup (listen=0, + like=1, dislike=2, unlike=3, undislike=4). +2. **Splits** events temporally — 300 train days, 30-min gap, 1 test day — by + Global Temporal Split (GTS). +3. **Segments** per-user event timelines into sessions on a 30-min inactivity + gap. +4. **Computes** per-item popularity for downstream metric weighting. +5. **Writes** the layout `DLRMv3YambdaDataset` expects: + +``` +/ +├── raw/5b/multi_event.parquet 50 GB (downloaded) +├── shared_metadata/ +│ ├── artist_item_mapping.parquet 60 MB +│ ├── album_item_mapping.parquet 76 MB +│ └── embeddings.parquet 18 GB (unused by HSTU training) +└── processed_5b/ + ├── train_sessions.parquet 47 GB ← main training input + ├── test_events.parquet 152 MB + ├── session_index.parquet 600 MB + ├── item_popularity.npy 75 MB + └── split_meta.json anchor + boundary stats +``` + +For smaller variants (`yambda-50m` / `yambda-500m`) substitute the dataset name +(`DATASET=yambda-50m ./download_dataset.sh`). Preprocessing takes ~2 min for 50m +and ~53 min for 5b end-to-end. + +Integrity is verified with `./verify_dataset.sh` against +[`md5sums_yambda_5b_processed.txt`](md5sums_yambda_5b_processed.txt). + +## 6. How data is fed to HSTU + +For every training anchor (a LISTEN event with >= `min_history` prior events), +the dataset builds a `(uih_kjt, candidate_kjt)` pair: + +``` +UIH (User Interaction History): + ┌─ Sequence features (chronologically interleaved across 3 pools) + │ item_id, artist_id, album_id ← per-position + │ action_weight ← per-position (LP_BIT/LIKE_BIT/SKIP_BIT) + │ action_timestamp, dummy_watch_time ← per-position + └─ Contextual features (length 1 each) + uid + 7 cross-feature hashes (user_x_artist, item_x_hour, …) + = 8 contextual entries + +CANDIDATE (the LISTEN event at the anchor): + item_id, artist_id, album_id, item_query_time, + item_action_weight (LP_BIT if listen_plus, else 0), + item_dummy_watchtime +``` + +The candidate's `action_weight` is **the supervision label**: HSTU's +`_get_supervision_labels_and_weights` masks BCE training to +`(supervision_bitmask & task_weight) > 0`, with `task_weight = 1` (LP bit) for +the single `listen_plus` task — so only listen_plus candidates supervise. + +### 6.1 Per-pool gather (the cap = L // 3 strategy) + +The UIH is built by `DLRMv3YambdaDataset._gather_interleaved_history`. For each +anchor it: + +1. Scans the most recent `scan_window` (default 20,000) events of any type + before the anchor, **clipped to user_start**. +2. From those, takes **the last `L // 3` events** from each of the three pools + (lp, like, skip) independently. +3. Concatenates and **re-sorts chronologically** to produce an interleaved + sequence. +4. Tags each event's pool identity into `action_weight` via OR'd bitmask + (LP=1, LIKE=2, SKIP=4). + +With `history_length = 4086` and `max_seq_len = 4096`: per-pool cap = `4086 // +3 = 1362`, and `3 × 1362 + 8 contextual + 1 candidate = 4095 <= 4096` (no +truncation). Because the `like` pool is rare (1.9%) it under-fills (~105 events +per anchor on average); the Triton jagged-attention backend skips unfilled +slots, so the under-fill costs sequence budget but not GPU compute. + +## 7. Optimizer + +Two optimizers, configured in +[`yambda_5b.gin`](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin): + +| component | optimizer | gin binding | key settings | +|---|---|---|---| +| Dense params (HSTU blocks, MLPs) | **Adam** | `dense_optimizer_factory_and_class.*` | lr `DENSE_LR`, betas (0.95, 0.999), eps 1e-8, weight_decay 0 | +| Sparse embedding tables | **RowWiseAdagrad** (fused FBGEMM TBE) | `sparse_optimizer_factory_and_class.*` | lr `SPARSE_LR`, eps 1e-8, weight_decay 0 | + +Gradient clipping (`GRAD_CLIP_NORM`, default `max_norm=1.0`) is applied to the +dense parameters on the streaming path; the fused sparse optimizer is +unaffected. Training is bf16 mixed precision (`make_model.bf16_training=True`). + +## 8. Hyperparameters + +All tunable hyperparameters live in +[`yambda_5b.gin`](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin) +(the config-file source of truth) and are **overridable via environment +variables** (the env value takes precedence over the gin default, per MLPerf +CONTRIBUTING rule 4d). The gin macros (`@env_int`, `@env_float`, `@env_str`) +enforce the correct type for each parameter. + +| hyperparameter | env var | gin binding | type | default | tuning rule | +|---|---|---|---|---|---| +| Per-rank batch size | `BATCH_SIZE` | `batch_size` | int | 1024 | positive integer (global batch = `BATCH_SIZE × world_size`) | +| Dense learning rate | `DENSE_LR` | `dense_optimizer_factory_and_class.learning_rate` | float | 1e-7 | positive float | +| Sparse learning rate | `SPARSE_LR` | `sparse_optimizer_factory_and_class.learning_rate` | float | 1e-7 | positive float | +| Grad clip max-norm | `GRAD_CLIP_NORM` | `streaming_train_eval_loop.grad_clip_norm` | float | 1.0 | float >= 0 (0 disables) | +| RNG seed | `SEED` | `seed_everything.seed` | int | 1 | any integer (-1 = random per run) | +| HSTU attention layers | `HSTU_NUM_LAYERS` | `get_hstu_configs.hstu_attn_num_layers` | int | 3 | positive integer | +| UIH history length | `HISTORY_LENGTH` | `get_dataset.history_length` | int | 4086 | positive integer (per-pool cap = L//3) | +| Max sequence length | `MAX_SEQ_LEN` | `get_hstu_configs.max_seq_len` | int | 4096 | positive integer (>= `history_length + 9`) | +| History strategy | `HISTORY_STRATEGY` | `get_dataset.history_strategy` | str | `interleaved` | one of `interleaved` \| `last_n` | +| Min history (anchor floor) | `MIN_HISTORY` | `get_dataset.min_history` | int | 4086 | integer >= 0 | +| Train user split | `TRAIN_SPLIT_PERCENTAGE` | `*.train_split_percentage` | float | 1.0 | float in (0, 1] | +| Streaming shuffle fraction | `STREAMING_SHUFFLE_FRACTION` | `get_dataset.streaming_shuffle_fraction` | float | 0.0 | float in [0, 1] | +| Streaming shuffle seed | `STREAMING_SHUFFLE_SEED` | `get_dataset.streaming_shuffle_seed` | int | 0 | any integer | +| Split salt | `SPLIT_SALT` | `get_dataset.split_salt` | int | 0 | any integer | +| Start window | `START_TS` | `streaming_train_eval_loop.start_ts` | int | 150 | integer >= 0 | +| Number of train windows | `NUM_TRAIN_TS` | `streaming_train_eval_loop.num_train_ts` | int | 149 | positive integer (clamped to available) | +| Sparse A2A fwd precision | `SPARSE_A2A_FWD` | `make_optimizer_and_shard.sparse_a2a_forward_precision` | str | `fp32` | one of `fp32` \| `bf16` \| `fp16` | +| Sparse A2A bwd precision | `SPARSE_A2A_BWD` | `make_optimizer_and_shard.sparse_a2a_backward_precision` | str | `fp32` | one of `fp32` \| `bf16` \| `fp16` | + +Non-tunable / fixed reference values (optimizer betas (0.95, 0.999), eps 1e-8, +weight_decay 0, bf16 training, streaming window = 86400 s) are pinned in the gin +file. Submitters tuning hyperparameters must follow the allowed values above and +the +[MLPerf training rules](https://github.com/mlcommons/training_policies/blob/master/training_rules.adoc#hyperparameters). + +## 9. Quality target & evaluation + +- **Metric**: AUC on the held-out future evaluation window (`window_auc` for the + `listen_plus` task), computed by `MetricsLogger` in + `generative_recommenders/dlrm_v3/utils.py`. +- **Target**: **eval AUC >= 0.75**. Set via `AUC_THRESHOLD=0.75` + (`MetricsLogger.auc_threshold`); the run logs `RUN_STOP` with `SUCCESS` and + stops once the target is reached. The gin default of `1.0` is unreachable + (trains all windows with no early stop) and is overridden by the reference + scripts. +- **Evaluation frequency**: the full-reference run uses + `EVAL_EVERY_DATA_PCT=0.005` — evaluate every 0.5% of the training stream + (~200 evenly-data-spaced eval points), independent of node count. The + alternative per-window cadence (`EVAL_EVERY_N_WINDOWS`) is mutually exclusive. +- **Evaluation set**: a fixed held-out future window (`eval_holdout_ts`, default + `start_ts + num_train_ts`); with `TRAIN_SPLIT_PERCENTAGE < 1.0` the held-out + users' anchors over that window form the eval set. The temporal one-window + lead guarantees no future leakage (see §11). + +Evaluation is always one window ahead of training, so reported AUC is genuine +next-period generalization. + +## 10. Reference Convergence Points (RCP) + +**Status: convergence logs generated for 3 batch sizes.** Per the MLPerf +[CONTRIBUTING guidance](https://github.com/mlcommons/training_policies/blob/master/CONTRIBUTING.md), +RCPs must cover at least 3 reasonable batch sizes using at least 2N seeds +(N = number of submission runs). An initial sweep has now been produced and +merged under [`rcp_logs/`](rcp_logs/) — **3 global batch sizes × 20 seeds each +(60 runs total; 2N with N = 10)** in MLPerf MLLOG format +(`submission_org=AMD`, `submission_platform=MI350`, `submission_benchmark=hstu`, +`submission_division=closed`). All runs use `opt_name=Adam` (dense) + +`opt_sparse_name=RowWiseAdagrad`, `gradient_accumulation_steps=1`. The LR +schedule keeps the **starting LR fixed at `0`** and the **warmup steps fixed at +`24000`**; only the **target (post-warmup) LR scales with global batch size**, +linearly from `1e-6` (gbs 8192) to `4 * 1e-6` (gbs 32768) — i.e. `1e-6`, +`2e-6`, `4e-6` for the three batch sizes. Evaluation is run at a **fixed 0.1% +data eval frequency** (`EVAL_EVERY_DATA_PCT=0.001`) across all runs. + +**Eval target: `eval_accuracy` (AUC) `>= 0.75`.** All 60 runs cross this +threshold, so the sweep gives a complete samples-to-converge distribution at +every batch size: + +| Global batch size | Dir | Seeds | Seeds reaching AUC >= 0.75 | Samples-to-converge (min / mean / max) | CV (samples) | Epoch-to-converge (min / mean / max) | Wall-clock to AUC 0.75 (min / mean / max) | +|---|---|---|---|---|---|---|---| +| 8192 | [`rcp_logs/gbs_8192/`](rcp_logs/gbs_8192/) | 20 | 20/20 | 61.9M / 69.3M / 80.3M | 5.6% | 0.027 / 0.030 / 0.035 | 2h22m / 2h39m / 2h59m | +| 16384 | [`rcp_logs/gbs_16384/`](rcp_logs/gbs_16384/) | 20 | 20/20 | 78.0M / 87.7M / 100.9M | 7.2% | 0.034 / 0.038 / 0.044 | 1h39m / 1h52m / 2h12m | +| 32768 | [`rcp_logs/gbs_32768/`](rcp_logs/gbs_32768/) | 20 | 20/20 | 98.6M / 113.0M / 128.5M | 6.7% | 0.043 / 0.049 / 0.056 | 1h17m / 1h23m / 1h36m | + +Each seed records the full `eval_accuracy` curve vs. `samples_count`; +"samples-to-converge" is the first eval crossing AUC `>= 0.75`. Epoch-to-converge +is that sample count as a fraction of one full training epoch +(`train_samples = 2,290,835,423`, e.g. 50% of samples = 0.5 epoch), taken from +the log's `epoch_num`. As expected, samples- and epoch-to-converge grow with +global batch size (convergence is reached well within the first ~5% of an epoch). +"CV (samples)" is the coefficient of variation (sample stddev / mean) of +samples-to-converge across the 20 seeds — all under ~7%, i.e. low run-to-run +variance. + +### 10.1 Tunable hyperparameters (benchmark submission) + +For a benchmark submission, only the following hyperparameters may be tuned: + +- **Target (post-warmup) learning rate** — the peak LR the warmup ramps up to. +- **Global batch size** — e.g. the 8192 / 16384 / 32768 points swept above. + +The rest of the parameters, including the following, are **fixed and not tunable**: + +- **Starting LR = `0`** (the warmup always begins from zero). +- **Warmup steps = `24000`**. +- **`skip_eval = 10`** — the first 10 eval rounds are skipped before convergence + is checked; fixed for submission. + + + + + + +## 11. Streaming (temporal-order) training + +`scripts/launch_slurm.sh` and `run_and_time.sh` default to +`--mode streaming-train-eval`, which trains Yambda in strict wall-clock order +instead of shuffling the whole corpus. The timeline is sliced into +fixed-duration **windows** (default 1 day, +`get_dataset.streaming_window_seconds = 86400`), and the loop walks them forward: + +``` +window T: train window T+1: eval (then train) window T+2: eval (then train) ... + └─ train window T ─┐ + └─ eval window T+1 ─┐ + └─ train window T+1 ─┐ + └─ eval window T+2 ... +``` + +i.e. for each step it **trains window T, then evaluates window T+1** before +advancing — always predicting the immediate future from the past. + +### 11.1 Temporal guarantee + +The streaming path enforces **no future leakage** at two levels: + +1. **Across windows** — a window is the set of anchors whose target/candidate + timestamp falls in `[t_min + T·W, t_min + (T+1)·W)`. Training only ever sees + windows `<= T`; the evaluation window `T+1` is strictly in the future of every + training anchor it is scored against. +2. **Within an anchor** — history is gathered **causally**: the UIH scan is + `scan_start:flat_pos` (events strictly before the anchor), so no event at or + after the anchor's timestamp can enter its features. + +This is a *temporal* split on the training stream — distinct from the +preprocessing GTS split (§5) that carves off the final test day. Windows are +indexed off the per-anchor target timestamp via a lazily-built, mmap'd +`anchor_ts` cache keyed by `(history_length, min_history)`. + +### 11.2 Streaming knobs + +All configurable via +[`yambda_5b.gin`](generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin) with +env overrides: + +| env | gin | default | meaning | +|---|---|---|---| +| `START_TS` | `streaming_train_eval_loop.start_ts` | 150 | first window (early windows are near-empty warm-up) | +| `NUM_TRAIN_TS` | `streaming_train_eval_loop.num_train_ts` | 149 | number of train windows (clamped to available) | +| `PERSISTENT_LOADER` | `streaming_train_eval_loop.persistent_loader` | 1 | reuse one worker pool across windows | +| `DOUBLE_BUFFER` | `streaming_train_eval_loop.double_buffer` | 1 | prepare the next window in a background thread | +| `EVAL_EVERY_N_WINDOWS` | `streaming_train_eval_loop.eval_every_n_windows` | 1 | eval cadence by window count (0 to use data-pct) | +| `EVAL_EVERY_DATA_PCT` | `streaming_train_eval_loop.eval_every_data_pct` | 0.0 | eval cadence by fraction of train data (full ref: 0.005) | +| `MIN_HISTORY` | `get_dataset.min_history` | 4086 | anchor-eligibility floor (0 = ~all users incl. cold-start) | + +### 11.3 Checkpointing & resume + +The streaming loop is resume-aware: set `CKPT_PATH` to enable DMP checkpoint +save/load (auto-resolves to the highest-numbered subdir), with retention via +`KEEP_LAST_N` and cadences `IN_WINDOW_CKPT_FREQ` / `CKPT_STEP_FREQ` / +`CKPT_TIME_INTERVAL_S`. The MLPerf run state (run-started flag, global sample +count) is persisted across resume so compliance logging is continuous. See +`generative_recommenders/dlrm_v3/checkpoint.py`. + +## 12. License + +Apache 2.0 (inherited from upstream). diff --git a/recommendation_v4/docs/multi_node_config.md b/recommendation_v4/docs/multi_node_config.md new file mode 100644 index 000000000..52fdbbb69 --- /dev/null +++ b/recommendation_v4/docs/multi_node_config.md @@ -0,0 +1,230 @@ +# Multi-Node Training Enablement (yambda-5b, MI350X / Broadcom bnxt_re RoCE) + +How N-node (N×8-GPU) distributed training was brought up for the yambda-5b HSTU +ranker on the `meta64` cv350 cluster, the hard problems solved, and **exactly +which settings are cluster/fabric-specific** so this can be reused or re-tuned +when the underlying network changes. + +Companion to [`perf_opt.md`](./perf_opt.md) and [`training_recipe.md`](./training_recipe.md). +The single entry point is [`scripts/launch_slurm.sh`](../scripts/launch_slurm.sh); +the Python side is `generative_recommenders/dlrm_v3/train/{train_ranker,utils}.py`. + +--- + +## TL;DR + +- Multi-node works over **real RDMA** (RoCEv2 on 8× Broadcom bnxt_re HCAs). + 2-node = `world_size=16`, clean `rc=0`, ~7.7–8.0k `global_sps` (≈1.28× of + 1-node 6.2k; weak scaling, per-rank batch fixed). +- The one non-obvious blocker was a **userspace RDMA provider ABI mismatch** + inside the container, fixed with an `LD_PRELOAD`/`LD_LIBRARY_PATH` **overlay** + of the host's matched `rdma-core` (no container lib surgery). +- Everything is one script with three auto-detected phases + (`orchestrate` → `provision` → `worker`) plus small Python changes for global + ranks. All cluster-specific knobs are env-overridable and tagged + `[CLUSTER-SPECIFIC]` in the script. + +--- + +## Architecture: one script, three phases + +`launch_slurm.sh` self-dispatches by context (`LAUNCH_SLURM_PHASE`, else +auto-detected via `/.dockerenv`): + +| Phase | Runs on | Does | +|---|---|---| +| `orchestrate` | SLURM batch host | Resolve rendezvous (`MASTER_ADDR/PORT`), ensure container on every node (calls `provision`), then `docker exec` the `worker` phase on every node (one srun task per node). | +| `provision` | each compute node (host) | Ensure the `yambda_primus` container is up (baked image if present, else base image + pip), stage the host RDMA overlay on NFS. | +| `worker` | inside the container | Derive topology, set NCCL/RDMA env, apply the RDMA overlay, spawn this node's 8 GPU ranks via `train_ranker`. `NNODES==1` => legacy single-node path unchanged. | + +Why one script: multi-node enablement is then a single committable file. The +worker phase is also what the streaming-e2e supervisor invokes directly +(single-node, already inside the container), so the production path is unchanged. + +``` +sbatch --nodes=N launch_slurm.sh + │ (batch host: orchestrate) + ├─ srun: provision ──> docker container up + RDMA overlay staged (×N nodes) + └─ srun: docker exec launch_slurm.sh (worker) (×N nodes) + │ in container: topology + NCCL/RDMA env + LD overlay + └─ python train_ranker ──> 8 local ranks ──> RCCL rendezvous over RDMA +``` + +--- + +## The hard problems (lessons learned) + +### 1. RDMA provider ABI mismatch — the core blocker + +**Symptom:** multi-node RCCL died at init with +`ibv_create_qp ... Bad address`. + +**Root cause:** the container image (`rocm/primus:v26.3`) ships an **older** +userspace `rdma-core` (v34, `libbnxt_re-rdmav34.so`) than the **host kernel** +bnxt_re driver's uapi (host `rdma-core` v61 / `libbnxt_re-rdmav59.so`). The v34 +provider enumerates the HCAs and creates *shallow* QPs fine, but **faults when +creating a deep send queue** — RCCL uses `max_send_wr=256`. Verified with a +parameterized verbs probe: v34 `create_qp` is OK at depth ≤16 and faults at ≥64; +the host v59 provider works at **every** depth. So it is purely the **userspace +provider**, not the kernel or the fabric (a 2-node RoCEv2 RDMA-write test passes +on the stock stack, and bare-metal RCCL benchmarks run fine with the host libs). + +**Fix (no container surgery):** the `provision` phase stages the host's matched +`rdma-core` on shared NFS (`$OVERLAY`): + +``` +$OVERLAY/lib/libibverbs.so.1 # host libibverbs v61 +$OVERLAY/lib/libibverbs.so -> .so.1 # UNVERSIONED symlink (critical, see below) +$OVERLAY/lib/libnl-3.so.200, libnl-route-3.so.200 +$OVERLAY/lib/libibverbs/.so # incl. libbnxt_re-rdmav59.so +``` + +The `worker` phase makes RCCL load it at runtime: + +```bash +export LD_LIBRARY_PATH="$OVERLAY/lib:$OVERLAY/lib/libibverbs:$LD_LIBRARY_PATH" +export LD_PRELOAD="$OVERLAY/lib/libibverbs.so.1:$LD_PRELOAD" +``` + +We do **not** modify the container's system libs — only this process tree's +`LD_*`. Single-node and other users keep the stock stack. + +### 2. The UNVERSIONED `libibverbs.so` symlink is mandatory + +An earlier overlay attempt set `LD_LIBRARY_PATH` but still failed with +`Bad address`. Reason: at `import torch` the ROCm stack pulls in the +**unversioned** soname `libibverbs.so` (not `libibverbs.so.1`). If the overlay +only has `libibverbs.so.1`, that unversioned lookup misses the overlay, falls +through to the **container's** old lib, which then occupies the `libibverbs.so.1` +slot — so RCCL's later `dlopen("libibverbs.so.1")` binds the v34 stack and +`create_qp(256)` faults again. The overlay **must** expose +`libibverbs.so -> libibverbs.so.1`. With it (verified via `/proc//maps`), +the process maps **only** the host lib. `LD_PRELOAD` is belt-and-braces so the +host lib claims the soname slot first. + +### 3. Two network planes — pin TCP bootstrap, RDMA for data + +The container is `--network=host`, so RCCL sees **all** host interfaces and, left +to auto-detect, picks the wrong one. These nodes expose: +- `benic1p1..benic8p1` — per-GPU point-to-point RoCE links on `192.168.{1..8}.x/31`. + These are **not node-routable** for plain TCP; the very first bring-up **hung** + in `init_process_group` because RCCL tried the TCP bootstrap over a + non-routable `192.168.x` backend addr. +- `fenic0` — the routable front-end (`10.190.x`). + +So we split the planes explicitly: +- `NCCL_SOCKET_IFNAME=fenic0` → TCP bootstrap/rendezvous over the routable NIC. +- `NCCL_IB_HCA=bnxt_re0..7` → RDMA **data** over the 8 RoCE HCAs (the RoCEv2 + fabric *is* reachable rail-to-rail at the RDMA layer even though plain IP is not). + +### 4. Minimal proven bnxt_re NCCL config + +The minimal set proven on these nodes (matches cmcknigh's bare-metal RCCL +benchmarks): `NCCL_IB_GID_INDEX=3` (RoCEv2 IPv4 GID), `NCCL_IB_TC=104` (RoCE +lossless / PFC traffic class). **Do not** add the heavy +`QPS_PER_CONNECTION / ECE / DMABUF` block — that belongs to a different +(ionic AINIC) fabric and is counterproductive on bnxt_re. GPU-Direct RDMA +(`NCCL_NET_GDR_LEVEL`) is left **off**: it needs DMABUF/peermem, unavailable +in-container here, so RCCL stages through host memory (still real RDMA). + +### 5. Rendezvous must be resolved on the host + +The container image has **no SLURM client** (`scontrol` absent). So the +`orchestrate` phase resolves `MASTER_ADDR` (first host of the allocation) and a +deterministic `MASTER_PORT` (`20000 + job_id % 20000`, same on all nodes) **on +the host** and forwards them into the container via `docker exec -e`. + +### 6. Global rank derivation (Python) + +`mp.start_processes` hands out a node-local `local_rank` (0..7). Every downstream +consumer (data sharding, checkpoint I/O, metrics) needs the **global** rank: + +```python +rank = node_rank * gpus_per_node + local_rank # train_ranker._main_func +device = torch.device(f"cuda:{local_rank}") # CUDA device stays node-local +``` + +Also: `make_optimizer_and_shard(local_world_size=gpus_per_node)` so the TorchRec +planner respects the intra-node GPU count, and `MetricsLogger(world_size=...)` +gets the live world size (the gin default of 8 would mis-normalize multi-node). +`NNODES==1` makes `rank == local_rank` — identical to the old single-node path. + +### 7. `$0` is the staged `slurm_script`, not the repo path + +For an sbatch batch script, `$0` = +`/var/spool/slurmd/job/slurm_script` (node-local), so deriving the script / +repo path from `$0` gives a path that **doesn't exist on other nodes** (`bash +$SELF` → "No such file", and the worker's `cd $REPO` → exit 127). The +`orchestrate` phase instead resolves the real shared-NFS path from SLURM: + +```bash +SCRIPT_PATH=$(scontrol show job "$SLURM_JOB_ID" | grep -oP 'Command=\K\S+') +# fallbacks: $SLURM_SUBMIT_DIR/scripts/launch_slurm.sh, then $SELF +REPO=$(cd "$(dirname "$SCRIPT_PATH")/.." && pwd) +``` + +### 8. `srun ... bash -c "…"` host-vs-remote expansion + +Inside the double-quoted srun command string, **plain `$VAR` expands now on the +batch host** (values computed in orchestrate: `$MASTER_ADDR`, `$SCRIPT_PATH`, …) +while **`\$VAR` is deferred to each compute node** (`\$SLURM_NODEID`, +`\$(hostname)`) where the per-node SLURM env lives. Mixing these up sends every +rank the wrong node id. + +### 9. `memlock` ulimit for QP registration + +`docker run --ulimit memlock=-1:-1` is **required** — RDMA QP memory +registration needs unlimited locked memory. A container started with the default +8 MB memlock fails QP creation regardless of the overlay. + +### 10. Provisioning & the image-bake caveat + +Fresh nodes otherwise re-download a **6.1 GB** ROCm torch wheel + pip + build +torchrec-from-git every time. The script supports a pre-baked image +(`docker commit` → NFS tar → `docker load` offline). **Caveat:** the committed +image is **~127 GB** (ROCm base is huge), so the full-image NFS tar is impractical +(loading it can be slower than re-downloading 6 GB). For true download-avoidance +prefer a **local pip wheelhouse** (`pip install --no-index --find-links` from +~8 GB of NFS wheels) or a **local registry** (ships only the ~35 GB delta layer). +The bake hook is left in (`BAKE_IMAGE=1`) but defaults off; provisioning falls +back to base-image + pip. + +### Debunked theory (do not re-introduce) + +An earlier claim that the container's rdma-core was "too old → 0 devices / +Bad address" and needed an **in-place lib copy** was a red herring: the "0 +devices" came from a *broken in-place copy* of the host EL9 libs (mixing v34 +tooling that links `IBVERBS_PRIVATE_34` with host v61 libs breaks symbol-version +lookup). The stock container enumerates all 8 HCAs fine. The real issue is only +the deep-QP create path; the fix is the **LD overlay**, never in-place surgery. + +--- + +## Cluster-specific settings — change these when the fabric/hardware changes + +All are env-overridable and tagged `[CLUSTER-SPECIFIC]` in `launch_slurm.sh` +(`grep '\[CLUSTER-SPECIFIC\]' scripts/launch_slurm.sh`). + +| Setting | Default (meta64) | What it is | How to find the right value | +|---|---|---|---| +| `#SBATCH --partition` | `meta64` | scheduler partition | `sinfo` | +| bind mounts + default paths | `/home/chcai`, `/apps/chcai` | repo + scratch, **must be shared/NFS on all nodes** | `df -h`, cluster docs | +| `IMAGE` | `rocm/primus:v26.3` | base container (GPU arch + ROCm version) | vendor image registry | +| docker `--device` | `/dev/kfd /dev/dri` (AMD) | GPU passthrough | NVIDIA: `--gpus all` / nvidia runtime | +| `--ulimit memlock` | `-1` | locked mem for RDMA QP | keep `-1` for any RDMA fabric | +| `TORCH_IDX` / torch,vision,audio | `rocm7.2`, `2.12.0+rocm7.2` … | ROCm-version'd wheels | `download.pytorch.org/whl/` | +| `FBGEMM_WHL` | gfx950 wheel on NFS | GPU-arch fbgemm | build/stage per arch | +| `NCCL_SOCKET_IFNAME` | `fenic0` | **routable** host NIC for TCP bootstrap | `ip -br addr` (pick the routable one; NOT the per-GPU RDMA NICs) | +| `NCCL_IB_HCA` | `bnxt_re0..7` | RDMA HCA device names | `ibv_devices` (vendor: `mlx5_*`, `ionic_*`, …) | +| `NCCL_IB_GID_INDEX` | `3` | RoCEv2 IPv4 GID index | `show_gids` (v1/v2 & IPv4/IPv6 differ per port) | +| `NCCL_IB_TC` | `104` | RoCE lossless / PFC traffic class | fabric/switch admin | +| `RDMA_OVERLAY` (+ provider .so) | `/apps/chcai/rdma_host_el9_new` | host rdma-core overlay | only needed if container rdma-core < host kernel uapi; else set `RDMA_OVERLAY=` to disable. Stage the host's matching `/usr/lib64/libibverbs/.so` | + +**Different NIC vendor (e.g. Mellanox `mlx5`)** typically means: change +`NCCL_IB_HCA` names, re-check `NCCL_IB_GID_INDEX`/`NCCL_IB_TC`, and the RDMA +overlay is often **unnecessary** (Mellanox userspace in the image usually matches +the host) — set `RDMA_OVERLAY=` to skip it. + +**Emergency fallback:** `NCCL_NET_TRANSPORT=socket` disables IB and runs +allreduce over TCP (`fenic0`). Functional but ~100–200× slower; use only to +isolate a fabric problem. diff --git a/recommendation_v4/docs/perf_opt.md b/recommendation_v4/docs/perf_opt.md new file mode 100644 index 000000000..7799848ad --- /dev/null +++ b/recommendation_v4/docs/perf_opt.md @@ -0,0 +1,73 @@ +# Performance Optimizations — MI350X HSTU / OneTrans (yambda-5b, bs=1024, TRITON) + +Performance work for the 8× MI350X HSTU ranker on `yambda-5b` at `batch_size=1024` +with the **TRITON** HSTU kernel and bf16 training. Companion to +[`training_recipe.md`](./training_recipe.md) (environment + reproduction). + +Throughput numbers are global samples/sec across 8 GPUs (`global_sps`), measured +at steady state (instantaneous, computed from consecutive logged steps). + +--- + +## LN-dropout: multi-row, separated-RNG path on MI350 + +### What + +`_ln_mul_dropout_*` has two kernel variants: + +- **legacy** — single program per row, RNG fused inline (`_ln_mul_dropout_fwd`). +- **separated-RNG** — multiple rows per program, dropout mask precomputed once + and reused by the backward (`_ln_mul_dropout_fwd_rng` / + `_ln_mul_dropout_bwd_dx_du_rng`). + +The separated path was previously gated to Blackwell only (`is_sm100_plus()`). +MI350X (`gfx950`) benefits from the same structure, so the gate now also enables +it on MI350. + +### Where + +| file | change | +|---|---| +| `ops/utils.py` | `is_amd_mi350()` (gfx950 detect) + `use_separated_rng_ln_mul_dropout()` gate | +| `ops/triton/triton_hstu_linear.py` | dispatch LN-dropout fwd to the separated-RNG path when the gate is true | + +```python +# ops/utils.py +def use_separated_rng_ln_mul_dropout() -> bool: + return is_sm100_plus() or is_amd_mi350() +``` + +### Perf + +**+5.6% end-to-end → 14,222 global sps** (separated-RNG vs legacy fused, identical +config, full boost clocks — see the caveat below). + +--- + +## Caveat — GPU clock lock can mask all perf changes + +A node-level GPU clock lock will silently invalidate any benchmark on this +machine, so check it before trusting numbers. + +During this work all 8 GPUs were stuck in **`perf_determinism`** performance +level at **sclk 1093 MHz** (DPM level 1) while the real max is **2200 MHz** +(level 2) — despite 100% utilization, ~370 W of power headroom (629 / 1000 W), +and low temps (~50 °C). This was **not** thermal/power throttling; it was +leftover node state from a prior job. + +Effect: a **uniform ~1.87× slowdown of every Triton compute kernel** +(`2200 / 1093 ≈ 2.0×`), including kernels unrelated to any code change. It made +the LN-dropout fix above look like a regression until the clock state was found. + +### Detect + fix + +```bash +rocm-smi --showperflevel # expect "auto", not perf_determinism/manual/low +rocm-smi -d 0 --showclocks # expect sclk ~2000+ MHz under load +rocm-smi --setperflevel auto # restore boost +``` + +`scripts/launch_slurm.sh` (worker phase) now logs the perf level + a live `sclk` sample on +every launch, auto-restores `auto` if it finds a `perf_determinism`/`manual`/`low` +lock, and warns (to reset from the host) if it lacks permission inside the +container. **Always sanity-check `sclk ≈ 2000+ MHz` before trusting a benchmark.** diff --git a/recommendation_v4/docs/training_recipe.md b/recommendation_v4/docs/training_recipe.md new file mode 100644 index 000000000..cf31a9fff --- /dev/null +++ b/recommendation_v4/docs/training_recipe.md @@ -0,0 +1,203 @@ +# Training Recipe + +Reproducible environment + configuration for training HSTU / DLRM-v3 on the +`yambda-5b` dataset. + +--- + +## MI350X + +Single-node, 8× AMD **Instinct MI350X** (`gfx950`, ~288 GiB HBM3e each), HSTU +ranker on `yambda-5b` with the **TRITON** HSTU kernel and **bf16** +mixed-precision training. + +### Hardware / host + +| item | value | +|---|---| +| GPUs | 8× AMD Instinct MI350X (`gfx950`, ROCm 7.2.1) | +| Host CPU | AMD EPYC 9655 96-Core (192 cores × 2 threads) | + +### Container image + +``` +rocm/primus:v26.3 +``` + +### Dependency versions + +Aligned with the B200 path: same torch major.minor, same torchrec commit, +same fbgemm SHA. The image's native torch / torchvision / torchaudio / +torchrec / fbgemm_gpu are all replaced; only the image's triton stays. + +| package | version | install | +|---|---|---| +| **torch** | `2.12.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torch==2.12.0+rocm7.2` | +| **torchvision** | `0.27.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchvision` — ABI must match torch 2.12 | +| **torchaudio** | `2.11.0+rocm7.2` | `pip install --upgrade --no-deps --index-url https://download.pytorch.org/whl/rocm7.2 torchaudio` — ABI must match torch 2.12 | +| **triton** | `3.6.0` | image native, unchanged | +| **fbgemm_gpu** | `fbgemm_gpu_nightly_rocm-2026.6.2` (built from FBGEMM commit `10b77573`, same SHA as the B200 path) for `gfx950` | rebuild from source against the replaced torch. Build command: `python setup.py -j 32 bdist_wheel --build-target=default --build-variant=rocm -DHIP_ROOT_DIR=/opt/rocm -DAMDGPU_TARGETS=gfx950` | +| **torchrec** | `1.7.0a0+bf55480` (git tag `v2026.06.01.00`) | `pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00"` | +| **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows. Installed from a pre-staged local tarball by `scripts/launch_smoke_8gpu.sh` | + +### Training configuration + +From `generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin`: + +| parameter | value | gin binding | +|---|---|---| +| num_workers (dataloader) | 4 | `make_train_test_dataloaders.num_workers` | +| prefetch_factor | 8 | `make_train_test_dataloaders.prefetch_factor` | +| num_blocks | 1 | `make_train_test_dataloaders.num_blocks` | +| train_split_percentage | 0.90 | `make_train_test_dataloaders.train_split_percentage` | +| history_length (per-sample UIH budget) | 2039 | `get_dataset.history_length` | +| max_seq_len (attention budget) | 2048 | `get_hstu_configs.max_seq_len` | +| bf16 training | True | `make_model.bf16_training` | +| HBM cap (per GPU) | 260 GiB | `make_optimizer_and_shard.hbm_cap_gb` (env `HBM_CAP_GB`) | +| **triton autotune pinning** | **False (pinned)** | `apply_env_bootstrap.TRITON_FULL_AUTOTUNE` | +| dense optimizer | Adam, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `dense_optimizer_factory_and_class.*` | +| sparse optimizer | RowWiseAdagrad, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `sparse_optimizer_factory_and_class.*` | +| world_size | 8 | `MetricsLogger.world_size` | + +Effective global batch = `batch_size × world_size = 32 × 8 = 256` samples/step. + +### Environment variables + +| var | value | purpose | +|---|---|---| +| `HSTU_HAMMER_KERNEL` | `TRITON` | fast HSTU kernel (vs `PYTORCH` fallback) | +| `DLRM_DATA_PATH` | dataset root | overrides gin default `/apps/chcai/dlrm_data` | +| `HBM_CAP_GB` | (optional) | embedding planner HBM budget per GPU | +| `RUN_NAME` | run id | results dir → `results//` | +| `PYTORCH_CUDA_ALLOC_CONF` | `expandable_segments:True` | allocator headroom | +| `HIP_VISIBLE_DEVICES` / `CUDA_VISIBLE_DEVICES` | `0,1,2,3,4,5,6,7` | rank visibility | + +`TRITON_FULL_AUTOTUNE` is set automatically by the gin-driven bootstrap +(`generative_recommenders.dlrm_v3.train._env_bootstrap.apply_env_bootstrap`), +which runs in `train_ranker._main_func` BEFORE the triton kernel modules +import — so the gin file is the source of truth. + +### Measured performance + +| variant | steady-state ms/step | global sps | epoch ETA (3.23B anchors) | +|---|---|---|---| +| nightly + fp32 + PYTORCH attn (baseline) | ~190 | ~1340 | ~28 d | +| nightly + bf16 + TRITON attn | ~93 | ~2787 | ~13.4 d | +| primus + bf16 + TRITON attn | ~67.5 | ~3793 | ~9.9 d | +| primus + fbgemm HEAD + bf16 + TRITON, autotune drift | ~53 fast / ~70 slow | 3700–4860 | 7.7–10.2 d | +| **primus + fbgemm HEAD + bf16 + TRITON + pinning (default)** | **~52** | **~4970** | **~7.6 d** | + +The "pinning" line is the deterministic per-cold-start equilibrium — +three layer-norm / jagged triton kernels have two stable autotune winners +and the pin forces the fast one every run. + +### Known pitfalls + +- The image ships `fbgemm_gpu==2026.5.14`. The wheel built from FBGEMM HEAD + (`2026.6.1`) is required for the 70 → 52 ms step. Build inside the + container so the wheel links against the image's native torch. +- Stock `polars` silently overflows on `yambda-5b` (> 4.29 B rows); always + use `polars-u64-idx`. +- When changing shape (batch size, history length), GPU, or triton/torch + version, flip `apply_env_bootstrap.TRITON_FULL_AUTOTUNE = True` and run + with `TRITON_PRINT_AUTOTUNING=1` to re-capture winners, then update the + pinned configs at the `pinned_or_full(...)` call sites in + `generative_recommenders/ops/triton/`. +- Do not run with bf16 on the `PYTORCH` HSTU attention backend at our + sequence length — `pt_hstu_attention`'s QK einsum backward overflows in + bf16 at N > 1k and produces NaN at step 1. bf16 is only safe with TRITON. + +--- + +## B200 + +Single-node, 8× NVIDIA **B200** (Blackwell, `sm_100`, ~183 GiB HBM each), HSTU +ranker on `yambda-5b` with the **TRITON** HSTU kernel and **bf16** mixed-precision +training. + +### Hardware / host + +| item | value | +|---|---| +| GPUs | 8× NVIDIA B200 (`sm_100`, compute capability 10.0) | +| Host driver | 580.159.03 (reports CUDA 13.0) | +| Forward-compat userspace driver | `libcuda.so.595.58.03` (CUDA 13.2.1; engaged automatically by the NGC image) | + +### Container image + +``` +nvcr.io/nvidia/pytorch:26.04-py3 +``` + +Digest: `sha256:192d749b4d773610ec9e01c0443a9df545d196c412b7b8fd33bfa3da362a49e7` + +The image's native PyTorch is kept as-is and must not be reinstalled (so CUPTI +stays matched to the driver and `sm_100` support is preserved). + +`nvcr.io/nvidia/pytorch:26.01-py3` (torch `2.10.0a0` / CUDA 13.1, digest +`sha256:38ed2ecb2c16d10677006d73fb0a150855d6ec81db8fc66e800b5ae92741007e`) is +also validated and performance-equivalent — rebuild `fbgemm_gpu` against +whichever image's torch you run. + +### Dependency versions + +| package | version | notes | +|---|---|---| +| **torch** | `2.12.0a0+0291f960b6.nv26.04.48445190` (CUDA 13.2) | native to the image; not reinstalled | +| **triton** | `3.6.0` | native to the image; provides `triton.language.make_tensor_descriptor` (required by the TRITON HSTU path) | +| **fbgemm_gpu** | FBGEMM commit `10b775730212923f65f7b78f79b6a01d80cf3c29` (2026-06-01 `main`, CUDA 13.2, `sm_100`) | built from source against the native torch; public wheels are ABI-incompatible with the NGC torch. The built wheel is named `fbgemm_gpu_nightly-2026.6.1` — that version is the build date, not the source date, so always identify the build by the commit above. Build command: `TORCH_CUDA_ARCH_LIST=10.0 python setup.py bdist_wheel --build-target default --build-variant cuda --package_channel nightly --nvml_lib_path /usr/lib/x86_64-linux-gnu/libnvidia-ml.so` (~55 min — the `sm_100` TBE-forward kernels dominate via `ptxas`) | +| **torchrec** | `1.7.0.dev20260601+cu130` (nightly, tested) | installed `--no-deps` from `https://download.pytorch.org/whl/nightly/cu130`. Perf-neutral vs stable `1.4.0`; use `1.4.0` (latest stable) if you prefer a non-pre-release | +| **polars-u64-idx** | `1.33.1` | 64-bit row index — `yambda-5b` has > 4.29 B rows (overflows stock polars' 32-bit index) | +| CUPTI (for `torch.profiler`) | 13.2 (native) | matches the driver; the `+cu128` stack's CUPTI 12.8 fails on B200 (`CUPTI_ERROR_INVALID_DEVICE`) | + +Additional Python deps: +`xxhash`, `gin-config`, `absl-py`, `pandas`, `tensorboard`, `pyarrow`, `pyyaml`, +`tqdm`, `psutil`, `torchmetrics==1.0.3`, `tensordict`, `pyre-extensions`, +`iopath`, `typing-inspect`. + +### Training configuration + +From `generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin`: + +| parameter | value | gin binding | +|---|---|---| +| batch_size (train) | 32 | `make_train_test_dataloaders.batch_size` | +| eval_batch_size | 32 | `make_train_test_dataloaders.eval_batch_size` | +| num_workers (dataloader) | 4 | `make_train_test_dataloaders.num_workers` | +| prefetch_factor | 8 | `make_train_test_dataloaders.prefetch_factor` | +| num_blocks | 1 | `make_train_test_dataloaders.num_blocks` | +| train_split_percentage | 0.90 | `make_train_test_dataloaders.train_split_percentage` | +| history_length (per-sample UIH budget) | 2039 | `get_dataset.history_length` | +| max_seq_len (attention budget) | 2048 | `get_hstu_configs.max_seq_len` | +| bf16 training | True | `make_model.bf16_training` | +| HBM cap (per GPU) | 150 GiB | `make_optimizer_and_shard.hbm_cap_gb` (env `HBM_CAP_GB`) | +| **triton autotune pinning** | **True (full autotune)** | `apply_env_bootstrap.TRITON_FULL_AUTOTUNE` — the pinned configs are MI350X-specific, so B200 runs full autotune to find its own `sm_100` winners | +| dense optimizer | Adam, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `dense_optimizer_factory_and_class.*` | +| sparse optimizer | RowWiseAdagrad, lr 1e-3, betas (0.95, 0.999), eps 1e-8 | `sparse_optimizer_factory_and_class.*` | +| world_size | 8 | `MetricsLogger.world_size` | + +Effective global batch = `batch_size × world_size = 32 × 8 = 256` samples/step. + +### Environment variables + +| var | value | purpose | +|---|---|---| +| `HSTU_HAMMER_KERNEL` | `TRITON` | fast HSTU kernel (vs `PYTORCH` fallback) | +| `TORCH_CUDA_ARCH_LIST` | `10.0` | target `sm_100` for JIT / Triton compilation | +| `DLRM_DATA_PATH` | dataset root | overrides gin default `/apps/chcai/dlrm_data` | +| `HBM_CAP_GB` | `150` | embedding planner HBM budget per GPU | +| `RUN_NAME` | run id | results dir → `results//` | +| `PYTORCH_CUDA_ALLOC_CONF` | `expandable_segments:True` | allocator headroom | +| `TRITON_CACHE_DIR` | cache path | persist compiled Triton kernels across runs | +| `WORLD_SIZE` / `LOCAL_WORLD_SIZE` | `8` | mp.spawn rank count | + +### Known pitfalls + +- Never reinstall torch in this image — a cu12x wheel breaks CUPTI and may drop + `sm_100`. +- The `+cu128` stack (`torch==2.7.1+cu128` + `fbgemm-gpu==1.2.0+cu128` + + `torchrec==1.2.0+cu128`) runs on B200 but cannot profile GPU activity (CUPTI + 12.8 vs the 13.2 driver). +- Stock `polars` silently overflows on `yambda-5b` (> 4.29 B rows); always use + `polars-u64-idx`. +- `EmbeddingBoundsCheck ... Setting idx to zero` warnings are benign data clamps. diff --git a/recommendation_v4/download_dataset.sh b/recommendation_v4/download_dataset.sh new file mode 100755 index 000000000..d02f382dc --- /dev/null +++ b/recommendation_v4/download_dataset.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# MLPerf Training reference script: download + preprocess the dataset. +# +# Downloads the Yambda dataset from HuggingFace (yandex/yambda) and runs the +# preprocessing pipeline (event-type encoding, temporal GTS split, session +# segmentation, item-popularity counts) into the on-disk layout that +# DLRMv3YambdaDataset consumes. This is a thin wrapper over +# generative_recommenders.dlrm_v3.preprocess_public_data +# so the full reference data pipeline lives in one place. +# +# Usage: +# DLRM_DATA_PATH=/path/to/dlrm_data ./download_dataset.sh +# DATASET=yambda-50m DLRM_DATA_PATH=/path/to/dlrm_data ./download_dataset.sh +# +# Env: +# DATASET dataset variant (default: yambda-5b). One of +# kuairand-1k | kuairand-27k | yambda-50m | yambda-500m | yambda-5b +# DLRM_DATA_PATH destination data root (required). +set -euo pipefail + +DATASET="${DATASET:-yambda-5b}" +: "${DLRM_DATA_PATH:?Set DLRM_DATA_PATH to the destination data root}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${REPO_ROOT}" + +echo "[download_dataset] dataset=${DATASET} data-path=${DLRM_DATA_PATH}" +mkdir -p "${DLRM_DATA_PATH}" + +python3 -m generative_recommenders.dlrm_v3.preprocess_public_data \ + --dataset "${DATASET}" \ + --data-path "${DLRM_DATA_PATH}" + +echo "[download_dataset] done. Preprocessed layout under ${DLRM_DATA_PATH}:" +echo " raw/5b/multi_event.parquet" +echo " shared_metadata/{artist,album}_item_mapping.parquet, embeddings.parquet" +echo " processed_5b/{train_sessions,test_events,session_index}.parquet" +echo " processed_5b/item_popularity.npy, processed_5b/split_meta.json" +echo "[download_dataset] verify integrity with: ./verify_dataset.sh" diff --git a/recommendation_v4/generative_recommenders/README.md b/recommendation_v4/generative_recommenders/README.md new file mode 100644 index 000000000..e69de29bb diff --git a/recommendation_v4/generative_recommenders/common.py b/recommendation_v4/generative_recommenders/common.py new file mode 100644 index 000000000..2ff8edf80 --- /dev/null +++ b/recommendation_v4/generative_recommenders/common.py @@ -0,0 +1,513 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import abc +import copy +import os +from enum import Enum, unique +from typing import Any, Callable, List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.ops.utils import is_sm100_plus, is_sm90_plus +from torch.fx._symbolic_trace import is_fx_tracing +from torch.utils._python_dispatch import _get_current_dispatch_mode_stack + +# @manual=//triton:triton +from triton.runtime.autotuner import Autotuner + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + +try: + # @manual=//triton:triton + import triton.language.extra.tlx # type: ignore + + HAS_TLX = True +except ImportError: + HAS_TLX = False + +try: + from generative_recommenders.fb.triton_cc.utils import triton_cc + from hammer.ops.triton.utils import triton_autotune + from hammer.utils import is_dev_mode, set_dev_mode, set_verbose_level +except ImportError: + # pyre-ignore + def triton_cc(annotations): + # pyre-ignore + def decorator(fn): + return fn + + return decorator + + # pyre-ignore + def triton_autotune( + configs: List[triton.Config], + key: List[str], + # pyre-ignore + prune_configs_by=None, + # pyre-ignore + reset_to_zero=None, + # pyre-ignore + restore_value=None, + warmup: int = 25, + rep: int = 100, + ): + # pyre-ignore + def decorator(fn): + return Autotuner( + fn, + fn.arg_names, + configs, + key, + reset_to_zero, + restore_value, + pre_hook=None, + post_hook=None, + prune_configs_by=prune_configs_by, + warmup=warmup, + rep=rep, + ) + + return decorator + + DEV_MODE: bool = False + VERBOSE_LEVEL: int = 0 + + def set_dev_mode(val: bool) -> None: + global DEV_MODE + DEV_MODE = val + + def is_dev_mode() -> bool: + global DEV_MODE # noqa: F824 + return DEV_MODE + + def set_verbose_level(level: int) -> None: + global VERBOSE_LEVEL + VERBOSE_LEVEL = level + + def get_verbose_level() -> int: + global VERBOSE_LEVEL # noqa: F824 + return VERBOSE_LEVEL + + +@unique +class HammerKernel(Enum): + TRITON = "TRITON" + TLX = "TLX" + PYTORCH = "PYTORCH" + CUDA = "CUDA" + TRITON_CC = "TRITON_CC" + TRITON_INFERENCE = "TRITON_INFERENCE" + CUTEDSL = "CUTEDSL" + + +class HammerModule(torch.nn.Module, abc.ABC): + _is_inference: bool = False + _use_triton_cc: bool = True + _training_dtype: torch.dtype = torch.float32 + _hammer_kernel: Optional[HammerKernel] = None + + def __init__( + self, + is_inference: bool, + training_dytpe: torch.dtype = torch.float32, + use_triton_cc: bool = _use_triton_cc, + hammer_kernel: Optional[HammerKernel] = None, + ) -> None: + super().__init__() + self._is_inference = is_inference + self._training_dtype = training_dytpe + self._hammer_kernel = hammer_kernel + self._use_triton_cc = use_triton_cc + + def hammer_kernel(self) -> HammerKernel: + kernel = self._hammer_kernel + if kernel is not None: + return kernel + if self._is_inference and self._use_triton_cc: + return HammerKernel.TRITON_CC + else: + return HammerKernel.TRITON + + # pyre-ignore[2] + def recursive_setattr(self, name: str, value: Any) -> None: + for _, module in self.named_modules(): + if hasattr(module, name): + setattr(module, name, value) + + def set_use_triton_cc(self, use_triton_cc: bool) -> None: + self._use_triton_cc = use_triton_cc + self.recursive_setattr("_use_triton_cc", use_triton_cc) + + def set_is_inference(self, is_inference: bool) -> None: + self._is_inference = is_inference + self.recursive_setattr("_is_inference", is_inference) + + def set_training_dtype(self, training_dtype: torch.dtype) -> None: + self._training_dtype = training_dtype + self.recursive_setattr("_training_dtype", training_dtype) + + def set_hammer_kernel(self, hammer_kernel: HammerKernel) -> None: + self._hammer_kernel = hammer_kernel + self.recursive_setattr("_hammer_kernel", hammer_kernel) + + @property + def is_inference(self) -> bool: + return self._is_inference + + @property + def is_eval(self) -> bool: + return (not self._is_inference) and (not self.training) + + @property + def is_train(self) -> bool: + return (not self._is_inference) and self.training + + +def generate_sparse_seq_len( + size: int, + max_seq_len: int, + sparsity: float, + device: torch.device, +) -> torch.Tensor: + if sparsity == 0.0: + return torch.zeros(size=(size,), device=device, dtype=torch.int) + elif sparsity == 1.0: + return torch.ones(size=(size,), device=device, dtype=torch.int) * max_seq_len + elif sparsity >= 0.5: + min_seq_len: int = int((2 * sparsity - 1.0) * max_seq_len) + return torch.randint( + low=min_seq_len, + high=max_seq_len, + size=(size,), + device=device, + dtype=torch.int, + ) + else: + min_seq_len: int = 0 + max_seq_len: int = int(2 * sparsity * max_seq_len) + return torch.randint( + low=min_seq_len, + high=max_seq_len, + size=(size,), + device=device, + dtype=torch.int, + ) + + +def apply_sampling( + lengths: torch.Tensor, + alpha: float, + max_seq_len: int, +) -> torch.Tensor: + threshold = int(max_seq_len ** (alpha / 2)) + no_sample_prob = (max_seq_len**alpha) / torch.pow(lengths, 2) + users_to_sample = torch.logical_and( + lengths > threshold, + torch.rand_like(no_sample_prob) < 1 - no_sample_prob, + ) + lengths = torch.where(users_to_sample, threshold, lengths) + return lengths + + +nv_gpu_unavailable: Tuple[bool, str] = ( + not torch.cuda.is_available() or torch.cuda.device_count() == 0, + "CUDA is not available or no GPUs detected", +) +nv_gpu_available: bool = not nv_gpu_unavailable[0] + + +amd_gpu_unavailable: Tuple[bool, str] = ( + not torch.version.hip, + "AMD HIP not available or no GPUs detected", +) +amd_gpu_available: bool = not amd_gpu_unavailable[0] + +gpu_unavailable: Tuple[bool, str] = ( + not nv_gpu_available and not amd_gpu_available, + "CUDA/HIP is not available or no GPUs detected", +) + +gpu_available: bool = not gpu_unavailable[0] + +blackwell_tlx_unavailable: Tuple[bool, str] = ( + not is_sm100_plus() or not HAS_TLX, + "Skip TLX and blackwell only tests", +) + +tma_unavailable: Tuple[bool, str] = ( + not is_sm90_plus(), # noqa + "Skip TMA only tests", +) + + +def switch_to_contiguous_if_needed(x: torch.Tensor) -> torch.Tensor: + if torch.jit.is_scripting(): + if x.stride(-1) == 1: + return x + return x.contiguous() + if torch.compiler.is_compiling(): + # Tell Dynamo this data-dependent value is in the range (0, 10**9) + torch._check(x.size(0) > 0) + torch._check(x.size(0) < 10**9) + # FX cannot trace Python control flow over symbolic stride checks + # (`x.stride(-1) == 1`). For AOT-T lowering, conservatively emit the + # contiguous op instead of branching on a symbolic value. + if is_fx_tracing(): + return x.contiguous() + if x.stride(-1) == 1: + return x + return x.contiguous() + + +def cdiv(x: int, y: int) -> int: + return (x + y - 1) // y + + +def backend_allow_tf32() -> bool: + return True + + +BACKEND_ALLOW_TF32: bool = backend_allow_tf32() + + +def next_power_of_2(n: int) -> int: + """Return the smallest power of 2 greater than or equal to n""" + n -= 1 + n |= n >> 1 + n |= n >> 2 + n |= n >> 4 + n |= n >> 8 + n |= n >> 16 + n |= n >> 32 + n += 1 + return n + + +def _prev_power_of_2_bitwise(x: int) -> int: + """Return the largest power of 2 less than or equal to x.""" + x |= x >> 1 + x |= x >> 2 + x |= x >> 4 + x |= x >> 8 + x |= x >> 16 + x |= x >> 32 + return (x >> 1) + 1 + + +@torch.fx.wrap +def _prev_power_of_2_legacy(x: int) -> int: + if torch.compiler.is_compiling(): + # Re-write to make Dynamo happy + x_tensor = torch.scalar_tensor(x, dtype=torch.int64) # type: ignore[arg-type] + x_tensor_orig = x_tensor.clone() + out_val = next_power_of_2(int(x_tensor.item())) # type: ignore[arg-type] + out = torch.scalar_tensor(out_val, dtype=torch.int64) + return int(torch.where(torch.lt(x_tensor_orig, out), out // 2, out).item()) # type: ignore[return-value] + else: + out = next_power_of_2(x) + return out // 2 if out > x else out + + +prev_power_of_2: Callable[[int], int] = ( + _prev_power_of_2_legacy + if os.environ.get("PREV_POWER_OF_2_IMPL", "legacy") == "legacy" + else _prev_power_of_2_bitwise +) + + +STATIC_MAX_SEQ_LENS: List[int] = [] +USE_RUNTIME_MAX_SEQ_LEN: bool = False + + +def set_static_max_seq_lens(max_seq_lens: List[int]) -> None: + global STATIC_MAX_SEQ_LENS + STATIC_MAX_SEQ_LENS = copy.deepcopy(max_seq_lens) + STATIC_MAX_SEQ_LENS.sort() + + +def set_use_runtime_max_seq_len(use_runtime_max_seq_len: bool) -> None: + global USE_RUNTIME_MAX_SEQ_LEN + USE_RUNTIME_MAX_SEQ_LEN = use_runtime_max_seq_len + + +def autotune_max_seq_len(runtime_max_seq_len: int) -> int: + global USE_RUNTIME_MAX_SEQ_LEN # noqa: F824 + + if USE_RUNTIME_MAX_SEQ_LEN: + return prev_power_of_2(runtime_max_seq_len) + else: + if STATIC_MAX_SEQ_LENS == []: + return 1 + for max_len in STATIC_MAX_SEQ_LENS: + if max_len >= runtime_max_seq_len: + return max_len + return STATIC_MAX_SEQ_LENS[-1] + + +def fine_grained_autotune_max_seq_len(runtime_max_seq_len: int) -> int: + global USE_RUNTIME_MAX_SEQ_LEN # noqa: F824 + + if USE_RUNTIME_MAX_SEQ_LEN: + return _fine_grained_bucket_size(runtime_max_seq_len) + else: + if STATIC_MAX_SEQ_LENS == []: + return 1 + for max_len in STATIC_MAX_SEQ_LENS: + if max_len >= runtime_max_seq_len: + return max_len + return STATIC_MAX_SEQ_LENS[-1] + + +def _generate_fine_grained_buckets() -> List[int]: + buckets = [ + 1024, + 2048, + 4096, + 8192, + 12288, + 16384, + 24576, + 32768, + 40960, + 49152, + 65536, + 81920, + 98304, + ] + return buckets + + +@torch.fx.wrap +def _fine_grained_bucket_size(x: int) -> int: + if torch.compiler.is_compiling(): + x_tensor = torch.scalar_tensor(x, dtype=torch.int64) + buckets = torch.tensor(_generate_fine_grained_buckets(), dtype=torch.int64) + + mask = buckets >= x_tensor + valid_buckets = torch.where( + mask, buckets, torch.tensor(2**31 - 1, dtype=torch.int64) + ) + + result = torch.where(mask.any(), valid_buckets.min(), buckets[-1]) + + return int(result.item()) + else: + buckets = _generate_fine_grained_buckets() + + for bucket in buckets: + if x <= bucket: + return bucket + + return buckets[-1] + + +@torch.fx.wrap +def fx_unwrap_optional_tensor(optional: Optional[torch.Tensor]) -> torch.Tensor: + assert optional is not None, "Expected optional to be non-None Tensor" + return optional + + +@torch.fx.wrap +def fx_arange(len: int, device: torch.device) -> torch.Tensor: + return torch.arange(len, device=device) + + +@torch.fx.wrap +def fx_infer_max_len( + lengths: torch.Tensor, +) -> int: + # Do not call ".item()" to avoid unbacked symint problems for lowering + max_len = int(lengths.max()) + if not torch.jit.is_scripting() and torch.compiler.is_compiling(): + # Tell Dynamo this data-dependent value is in the range [0, 10**9) + torch._check_is_size(max_len) + torch._check(max_len < 10**9) + torch._check(max_len > 0) + return max_len + + +@torch.fx.wrap +def fx_mark_length_features(tensor: torch.Tensor) -> torch.Tensor: + return tensor + + +@torch.fx.wrap +def fx_torch_ones( + shape: List[int], + device: torch.device, + dtype: torch.dtype, +) -> torch.Tensor: + return torch.ones(shape, device=device, dtype=dtype) + + +@torch.fx.wrap +def fx_torch_zeros(shape: List[int], device: torch.device) -> torch.Tensor: + return torch.zeros(shape, device=device) + + +def _is_in_dispatch_modes(mode_names: List[str]) -> bool: + modes = _get_current_dispatch_mode_stack() + return any(mode.__class__.__name__ in mode_names for mode in modes) + + +def should_trigger_eager_impl() -> bool: + if torch.jit.is_scripting(): + return True + if torch.compiler.is_compiling(): + return False + return _is_in_dispatch_modes(["SplitDispatchMode", "FakeTensorMode"]) + + +@torch.fx.wrap +def jagged_to_padded_dense( + values: torch.Tensor, + offsets: List[torch.Tensor], + max_lengths: List[int], + padding_value: float, +) -> torch.Tensor: + return torch.ops.fbgemm.jagged_to_padded_dense( + values=values, + offsets=offsets, + max_lengths=max_lengths, + padding_value=padding_value, + ) + + +@torch.fx.wrap +def dense_to_jagged( + dense: torch.Tensor, + x_offsets: List[torch.Tensor], +) -> torch.Tensor: + return torch.ops.fbgemm.dense_to_jagged( + dense=dense, + x_offsets=x_offsets, + )[0] + + +def init_mlp_weights_optional_bias(m: torch.nn.Module) -> None: + if isinstance(m, torch.nn.Linear): + torch.nn.init.xavier_uniform_(m.weight) + if m.bias is not None: + m.bias.data.fill_(0.0) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py new file mode 100644 index 000000000..46cc10e2e --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/checkpoint.py @@ -0,0 +1,680 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +Checkpoint utilities for saving and loading DLRMv3 model checkpoints. + +This module provides functions for saving and loading distributed model checkpoints, +including both sparse (embedding) and dense (non-embedding) components. +""" + +import gc +import logging +import os +import random +import shutil +import time +from datetime import datetime +from typing import Any, Dict, Optional, Set, Tuple + +import gin +import numpy as np +import torch +from generative_recommenders.dlrm_v3.utils import ( + BinnedCumulativeAUC, + LifetimeAUCMetricComputation, + MetricsLogger, +) +from torch.distributed.checkpoint.stateful import Stateful +from torch.optim.optimizer import Optimizer +from torchrec.distributed.types import ShardedTensor + +logger: logging.Logger = logging.getLogger(__name__) + +# Sentinel meaning "the saved window completed in full" — when the loop reads +# this back it advances start_ts past the saved train_ts. Anything >=0 means the +# saved checkpoint stopped mid-window after K batches; resume continues that +# window at batch K. +WINDOW_COMPLETE: int = -1 + +# Filename (per-rank) holding the lifetime-AUC trailing buffers, mirroring the +# rng_rank{rank}.pt pattern. The buffers are per-rank-local, so a single +# rank-0 copy in non_sparse.ckpt would (wrongly) restore 1/world_size of the +# true history to every rank — hence a dedicated per-rank artifact. +METRICBUF_FILE_FMT: str = "metricbuf_rank{rank}.pt" + + +def _metric_blob_state_dict(m: torch.nn.Module) -> Dict[str, Any]: + """State dict for the shared (rank-0) non_sparse.ckpt metric blob. + + Both lifetime-AUC backends carry per-rank-local state that is persisted + authoritatively per-rank in ``metricbuf_rank{rank}.pt``; we must keep it out + of the shared blob so a rank's load doesn't inherit rank-0's counts: + + - ``LifetimeAUCMetricComputation``: drop the explicitly-serialized trailing + buffer keys (the rest of the blob keys are the parent's persistent state). + - ``BinnedCumulativeAUC``: zero the histogram buffers (they are persistent so + the keys must remain for a strict load, but the values are neutralized). + + All other metrics serialize normally. In both cases the per-rank file is + loaded afterward and is authoritative. + """ + sd = m.state_dict() + if isinstance(m, LifetimeAUCMetricComputation): + prefix = LifetimeAUCMetricComputation._LIFETIME_KEY_PREFIX + sd = {k: v for k, v in sd.items() if not k.startswith(prefix)} + elif isinstance(m, BinnedCumulativeAUC): + sd = { + k: (torch.zeros_like(v) if torch.is_tensor(v) else v) + for k, v in sd.items() + } + return sd + + +def _collect_perrank_metric_state( + metric_logger: "MetricsLogger", +) -> Dict[str, Dict[str, Any]]: + """Map "||" -> state_dict for every metric whose + cumulative state is per-rank-local and must be restored per-rank: + + - lifetime-AUC instances (`LifetimeAUCMetricComputation` trailing buffer, or + `BinnedCumulativeAUC` histograms) in class_metrics train/eval. Covers the + train lifetime AUC and, in legacy single-set eval, the eval lifetime AUC, + under either configured backend. + - the ENTIRE cumulative eval set (`eval_cum`, both class + regression) used + by the streaming dual-set eval: the lifetime-AUC backend state plus the + persistent cumulative scalar sums of NE/Accuracy/GAUC/MSE/MAE. + + Selected by structure/isinstance (not a hard index) since metric positions + depend on the configured tasks/mode. + """ + out: Dict[str, Dict[str, Any]] = {} + for mode in ("train", "eval"): + for idx, m in enumerate(metric_logger.class_metrics.get(mode, [])): + if isinstance(m, (LifetimeAUCMetricComputation, BinnedCumulativeAUC)): + out[f"class_metrics|{mode}|{idx}"] = m.state_dict() + for coll in ("class_metrics", "regression_metrics"): + for idx, m in enumerate(getattr(metric_logger, coll).get("eval_cum", [])): + out[f"{coll}|eval_cum|{idx}"] = m.state_dict() + return out + + +def _restore_perrank_metric_state( + metric_logger: "MetricsLogger", state: Dict[str, Dict[str, Any]] +) -> None: + for key, sd in state.items(): + coll, mode, idx_str = key.split("|") + getattr(metric_logger, coll)[mode][int(idx_str)].load_state_dict(sd) + + +def _perrank_sample_counts(metric_logger: "MetricsLogger") -> Dict[str, int]: + out: Dict[str, int] = {} + + def _count(m: torch.nn.Module) -> Optional[int]: + if isinstance(m, LifetimeAUCMetricComputation): + return m.lifetime_sample_count() + if isinstance(m, BinnedCumulativeAUC): + return m.cumulative_sample_count() + return None + + for mode in ("train", "eval", "eval_cum"): + for idx, m in enumerate(metric_logger.class_metrics.get(mode, [])): + n = _count(m) + if n is not None: + out[f"class|{mode}|{idx}"] = n + return out + + +class SparseState(Stateful): + """ + Stateful wrapper for sparse (embedding) tensors in a model. + + This class implements the Stateful interface for distributed checkpointing, + allowing sparse tensors to be saved and loaded separately from dense tensors. + + Args: + model: The PyTorch model containing sparse tensors. + sparse_tensor_keys: Set of keys identifying sparse tensors in the model's state dict. + """ + + def __init__(self, model: torch.nn.Module, sparse_tensor_keys: Set[str]) -> None: + self.model = model + self.sparse_tensor_keys = sparse_tensor_keys + + def state_dict(self) -> Dict[str, torch.Tensor]: + out_dict: Dict[str, torch.Tensor] = {} + is_sharded_tensor: Optional[bool] = None + for k, v in self.model.state_dict().items(): + if k in self.sparse_tensor_keys: + if is_sharded_tensor is None: + is_sharded_tensor = isinstance(v, ShardedTensor) + assert is_sharded_tensor == isinstance(v, ShardedTensor) + out_dict[k] = v + return out_dict + + def load_state_dict(self, state_dict: Dict[str, torch.Tensor]) -> None: + incompatible_keys = self.model.load_state_dict(state_dict, strict=False) + assert not incompatible_keys.unexpected_keys + + +def is_sparse_key(k: str, v: torch.Tensor) -> bool: + return isinstance(v, ShardedTensor) or "embedding_collection" in k + + +def load_dense_state_dict(model: torch.nn.Module, state_dict: Dict[str, Any]) -> None: + own_state = model.state_dict() + own_state_dense_keys = {k for k, v in own_state.items() if not is_sparse_key(k, v)} + state_dict_dense_keys = { + k for k, v in state_dict.items() if not is_sparse_key(k, v) + } + assert own_state_dense_keys == state_dict_dense_keys, ( + f"expects {own_state_dense_keys} but gets {state_dict_dense_keys}" + ) + for name in state_dict_dense_keys: + param = state_dict[name] + if isinstance(param, torch.nn.Parameter): + # backwards compatibility for serialized parameters + param = param.data + own_state[name].copy_(param) + + +def _rng_state(device: torch.device) -> Dict[str, Any]: + """Snapshot every RNG source bit-equal training depends on. + + HSTU has stochastic dropout (input_dropout=0.2, linear_dropout_rate=0.1) + consuming the per-device CUDA RNG cycle each step. Without round-tripping + these, a resumed run draws different dropout masks and the resumed AUC + trajectory diverges from the uninterrupted run within a few steps. + """ + return { + "cpu": torch.get_rng_state(), + "cuda": torch.cuda.get_rng_state(device), + "numpy": np.random.get_state(), + "random": random.getstate(), + } + + +def _restore_rng_state(state: Dict[str, Any], device: torch.device) -> None: + torch.set_rng_state(state["cpu"]) + torch.cuda.set_rng_state(state["cuda"], device) + np.random.set_state(state["numpy"]) + random.setstate(state["random"]) + + +def _list_numeric_subdirs(base_path: str) -> list[str]: + """Return subdir names of `base_path` that look like an int, sorted ascending. + + Filters out `*.tmp` (orphaned in-progress saves), `*.sparse/` and any other + non-numeric entries. + """ + if not os.path.isdir(base_path): + return [] + out: list[str] = [] + for name in os.listdir(base_path): + if name.isdigit(): + out.append(name) + return sorted(out, key=int) + + +def _resolve_latest_subdir(path: str) -> str: + """Map a base ckpt dir → its highest-numbered numeric subdir. + + Used so users can set `load_dmp_checkpoint.path = ""` (or + `CKPT_PATH=`) and automatically pick up the most recent save without + needing to know which step number to point at. If `path` already names a leaf save (numeric basename) it's returned + unchanged. If the base dir has no numeric subdirs yet — the cold-start case + where ``CKPT_PATH`` is configured but nothing has been saved (e.g. the + interrupt phase of the resume test starts from a freshly-cleaned dir) — we + return ``""`` so ``load_*_checkpoint`` no-ops instead of asserting on a + missing ``sparse/.metadata``. + """ + if not path: + return path + base = path.rstrip("/") + leaf = os.path.basename(base) + if leaf.isdigit(): + return base # already a leaf, caller knows what it wants + subs = _list_numeric_subdirs(base) + if not subs: + logger.info("No checkpoint subdirs under %s — cold start (no load).", base) + return "" # nothing to load → load_*_checkpoint short-circuits + resolved = os.path.join(base, subs[-1]) + logger.info("Auto-latest checkpoint: %s → %s", base, resolved) + return resolved + + +def _prune_old_checkpoints(base_path: str, keep_last_n: int, just_saved_subdir: str) -> None: + """Delete numeric subdirs older than the keep_last_n most recent. + + Defensive: never prune `just_saved_subdir` even if it would be evicted by + the keep_last_n window (shouldn't happen since we just wrote it, but + catches off-by-one bugs). Skipped entirely when keep_last_n<=0. + """ + if keep_last_n <= 0: + return + subs = _list_numeric_subdirs(base_path) + if len(subs) <= keep_last_n: + return + to_prune = subs[:-keep_last_n] + for name in to_prune: + full = os.path.join(base_path, name) + if os.path.realpath(full) == os.path.realpath(just_saved_subdir): + continue + try: + shutil.rmtree(full) + logger.info("Pruned old checkpoint: %s", full) + except OSError as e: + logger.warning("Failed to prune %s: %s", full, e) + + +def _cleanup_stale_tmps(base_path: str) -> None: + """Remove `*.tmp`/`*.old` subdirs left by a crashed prior save attempt. + + `*.tmp` = an interrupted write; `*.old` = an interrupted atomic-overwrite + swap (see the promotion step in save_dmp_checkpoint). Both are non-numeric + so `_resolve_latest_subdir` already ignores them; this just reclaims disk. + """ + if not os.path.isdir(base_path): + return + for name in os.listdir(base_path): + if name.endswith(".tmp") or name.endswith(".old"): + full = os.path.join(base_path, name) + try: + shutil.rmtree(full) + logger.warning("Removed stale checkpoint dir: %s", full) + except OSError as e: + logger.warning("Failed to remove stale dir %s: %s", full, e) + + +@gin.configurable +def save_dmp_checkpoint( + model: torch.nn.Module, + optimizer: Optimizer, + metric_logger: MetricsLogger, + rank: int, + batch_idx: int, + path: str = "", + keep_last_n: int = 1, + train_ts: Optional[int] = None, + batch_idx_in_window: int = WINDOW_COMPLETE, + device: Optional[torch.device] = None, + split_contract: Optional[Dict[str, Any]] = None, +) -> None: + """ + Save a distributed model checkpoint including sparse and dense components. + + Writes into a per-rank-coordinated atomic layout: + /.tmp/ ← directory written into during save + // ← atomically renamed from .tmp on success + + A crash mid-save leaves the `.tmp/` orphan, which `_cleanup_stale_tmps` + sweeps on the next save attempt and which `_resolve_latest_subdir` ignores + (non-numeric basename). The previous successful `/` remains valid. + + Args: + model: The model to checkpoint. + optimizer: The optimizer whose state should be saved. + metric_logger: The metrics logger containing training/eval metrics. + rank: The current process rank in distributed training. + batch_idx: Subdir name (for streaming we set this == train_ts so the + on-disk layout monotonically increases). + path: Base path for saving the checkpoint. If empty, no checkpoint is saved. + keep_last_n: Number of most-recent numeric subdirs to retain after a + successful save. Set 1 (default) for disk-bounded long runs; + <=0 disables pruning. + train_ts: For streaming-train-eval, the current train timestamp. + Stored in non_sparse.ckpt so resume knows which window to enter. + batch_idx_in_window: For streaming-train-eval, batches completed within + train_ts. WINDOW_COMPLETE (-1) means the window finished; resume + advances to train_ts+1. >=0 means crash happened mid-window; resume + re-enters train_ts at batch_idx_in_window. + device: CUDA device for the per-rank RNG snapshot. Required for + bit-equal trajectories across resume (HSTU dropout consumes the + per-device RNG cycle). + """ + if path == "": + return + # Exclude checkpoint wall-time from the train step-time window so step_ms + # reports canonical compute latency; the duration is surfaced separately + # (window_ckpt_time_ms + the per-save log below). pause/resume are no-ops if + # metric_logger is None. Not wrapped in try/finally: a save that raises + # crashes the process (supervisor restarts fresh), so a dangling pause on + # the soon-dead logger is irrelevant. + _t_ckpt_start = time.perf_counter() + if metric_logger is not None: + metric_logger.pause_perf("ckpt") + base_path = path + # Atomic-save layout: write to .tmp, rename to final, prune older. + tmp_subdir = f"{base_path}/{batch_idx}.tmp" + final_subdir = f"{base_path}/{batch_idx}" + + if rank == 0: + _cleanup_stale_tmps(base_path) + # Always (re)write into a fresh .tmp. An existing `final_subdir` with the + # same batch_idx (e.g. a later in-window save for the same train_ts, or a + # deterministic re-run at the same step) is overwritten atomically at the + # promotion step below — NOT skipped here. Skipping would desync ranks: + # the collective barrier/checkpoint.save calls below run on *every* rank, + # so a rank-0-only early return deadlocks ranks 1..N on the next barrier. + shutil.rmtree(tmp_subdir, ignore_errors=True) + os.makedirs(tmp_subdir, exist_ok=True) + os.makedirs(f"{tmp_subdir}/sparse/", exist_ok=True) + torch.distributed.barrier() + sparse_path = f"{tmp_subdir}/sparse/" + non_sparse_ckpt = f"{tmp_subdir}/non_sparse.ckpt" + + sparse_tensor_keys = { + k for k, v in model.state_dict().items() if isinstance(v, ShardedTensor) + } + if rank == 0: + dense_state_dict = { + k: v + for k, v in model.state_dict().items() + if not isinstance(v, ShardedTensor) + } + class_metric_state_dict = { + "train": [ + _metric_blob_state_dict(m) + for m in metric_logger.class_metrics["train"] + ], + "eval": [ + _metric_blob_state_dict(m) + for m in metric_logger.class_metrics["eval"] + ], + } + regression_metric_state_dict = { + "train": [ + m.state_dict() for m in metric_logger.regression_metrics["train"] + ], + "eval": [m.state_dict() for m in metric_logger.regression_metrics["eval"]], + } + torch.save( + { + "dense_dict": dense_state_dict, + "optimizer_dict": optimizer.state_dict(), + "class_metrics": class_metric_state_dict, + "reg_metrics": regression_metric_state_dict, + "global_step": metric_logger.global_step, + # MLPerf progress counter (global trained samples). Defaulted on + # load so pre-existing checkpoints restore as 0 and resume the + # count from there. + "cumulative_train_samples": metric_logger.cumulative_train_samples, + # MLPerf run-marker state: lets a resume relaunch continue the + # SAME run's event stream without re-emitting INIT_START/RUN_START. + "mlperf_run_started": metric_logger.mlperf_run_started, + "sparse_tensor_keys": sparse_tensor_keys, + # Streaming resume fields. Defaulted on load so old checkpoints + # (pre-streaming-resume) still load as a normal restart. + "train_ts": train_ts, + "batch_idx_in_window": batch_idx_in_window, + # Immutable train:eval split + resume-determinism contract + # (train_split_percentage, split_salt, eval holdout window, + # batch_size, world_size). Validated on resume so a relaunch + # cannot silently change the split (which would desync the skip + # offset and/or train on held-out eval users). None for + # non-holdout / legacy runs. + "split_contract": split_contract, + }, + non_sparse_ckpt, + ) + + # Per-rank RNG snapshot. Written even on a single rank because dropout's + # randomness comes from the CUDA generator which differs across devices. + if device is not None: + rng_path = f"{tmp_subdir}/rng_rank{rank}.pt" + torch.save(_rng_state(device), rng_path) + + # Per-rank cumulative metric state (lifetime-AUC buffers + cumulative-eval + # histograms/scalar sums). Written by EVERY rank (outside the rank-0 block) + # because this state is per-rank-local; restoring rank-0's copy to all ranks + # would lose (world_size-1)/world_size of the history. + if metric_logger is not None: + perrank_state = _collect_perrank_metric_state(metric_logger) + if perrank_state: + torch.save( + perrank_state, + f"{tmp_subdir}/{METRICBUF_FILE_FMT.format(rank=rank)}", + ) + logger.info( + "checkpoint save: cumulative metric state rank=%d samples=%s", + rank, + _perrank_sample_counts(metric_logger), + ) + + torch.distributed.barrier() + sparse_dict = {"sparse_dict": SparseState(model, sparse_tensor_keys)} + torch.distributed.checkpoint.save( + sparse_dict, + storage_writer=torch.distributed.checkpoint.FileSystemWriter(sparse_path), + ) + torch.distributed.barrier() + # Promote .tmp → final, then prune. Done on rank 0 only since the directory + # operations are global filesystem state. + if rank == 0: + if os.path.exists(final_subdir): + # POSIX rename() refuses to replace a non-empty directory, so we + # can't os.replace(tmp, final) directly. Swap the old snapshot aside + # (instant rename), move the new one into place, then delete the old. + # The `.old` name is non-numeric → ignored by _resolve_latest_subdir + # and swept by _cleanup_stale_tmps on the next save if we crash mid-swap. + old_aside = f"{final_subdir}.old" + shutil.rmtree(old_aside, ignore_errors=True) + os.replace(final_subdir, old_aside) + os.replace(tmp_subdir, final_subdir) + shutil.rmtree(old_aside, ignore_errors=True) + else: + os.replace(tmp_subdir, final_subdir) + _prune_old_checkpoints(base_path, keep_last_n, final_subdir) + logger.info( + "checkpoint successfully saved → %s (wall-time %.2fs)", + final_subdir, + time.perf_counter() - _t_ckpt_start, + ) + torch.distributed.barrier() + if metric_logger is not None: + metric_logger.resume_perf("ckpt") + + +@gin.configurable +def load_sparse_checkpoint( + model: torch.nn.Module, + path: str = "", +) -> None: + if path == "": + return + sparse_path = f"{path}/sparse/" + + sparse_tensor_keys = { + k for k, v in model.state_dict().items() if is_sparse_key(k, v) + } + sparse_dict = {"sparse_dict": SparseState(model, sparse_tensor_keys)} + gc.collect() + torch.distributed.checkpoint.load( + sparse_dict, + storage_reader=torch.distributed.checkpoint.FileSystemReader(sparse_path), + ) + gc.collect() + print("sparse checkpoint successfully loaded") + + +@gin.configurable +def load_nonsparse_checkpoint( + model: torch.nn.Module, + device: torch.device, + optimizer: Optional[Optimizer] = None, + metric_logger: Optional[MetricsLogger] = None, + path: str = "", + rank: int = 0, +) -> Tuple[Optional[int], int, Optional[Dict[str, Any]]]: + """ + Load non-sparse (dense) components from a checkpoint. + + Loads dense model parameters, and optionally optimizer state and metrics. + Also restores per-rank RNG state if a matching `rng_rank{rank}.pt` is found + next to `non_sparse.ckpt`. + + Returns: + (train_ts, batch_idx_in_window, split_contract) — the streaming resume + hint and the saved train:eval split contract (None for legacy / non- + holdout checkpoints). `(None, WINDOW_COMPLETE, None)` if not a streaming + checkpoint or no path supplied. + """ + if path == "": + return None, WINDOW_COMPLETE, None + non_sparse_ckpt = f"{path}/non_sparse.ckpt" + + # weights_only=False: these are our own trusted checkpoints, and they hold + # non-tensor objects (optimizer/metric state dicts, numpy-backed RNG state) + # that PyTorch>=2.6's weights_only=True default refuses to unpickle. + non_sparse_state_dict = torch.load( + non_sparse_ckpt, map_location=device, weights_only=False + ) + load_dense_state_dict(model, non_sparse_state_dict["dense_dict"]) + print("dense checkpoint successfully loaded") + if optimizer is not None: + optimizer.load_state_dict(non_sparse_state_dict["optimizer_dict"]) + print("optimizer checkpoint successfully loaded") + if metric_logger is not None: + metric_logger.global_step = non_sparse_state_dict["global_step"] + # Defaulted for legacy checkpoints written before the counter existed. + metric_logger.cumulative_train_samples = non_sparse_state_dict.get( + "cumulative_train_samples", 0 + ) + # Defaulted False for legacy/cold checkpoints: a resume that loads a + # checkpoint where the run was already open continues without re-emitting + # the run markers. + metric_logger.mlperf_run_started = non_sparse_state_dict.get( + "mlperf_run_started", False + ) + class_metric_state_dict = non_sparse_state_dict["class_metrics"] + regression_metric_state_dict = non_sparse_state_dict["reg_metrics"] + # Length-safe positional restore: if a checkpoint was written with a + # different metric set (e.g. tasks added/removed since), restore the + # overlap instead of crashing with an IndexError at run end. + def _restore_metric_list( + live: list, saved: Optional[list], label: str + ) -> None: + saved = saved or [] + if len(live) != len(saved): + logger.warning( + "metric count mismatch for %s: live=%d saved=%d; " + "restoring overlapping %d", + label, + len(live), + len(saved), + min(len(live), len(saved)), + ) + for i in range(min(len(live), len(saved))): + live[i].load_state_dict(saved[i]) + + _restore_metric_list( + metric_logger.class_metrics["train"], + class_metric_state_dict.get("train"), + "class/train", + ) + _restore_metric_list( + metric_logger.class_metrics["eval"], + class_metric_state_dict.get("eval"), + "class/eval", + ) + _restore_metric_list( + metric_logger.regression_metrics["train"], + regression_metric_state_dict.get("train"), + "reg/train", + ) + _restore_metric_list( + metric_logger.regression_metrics["eval"], + regression_metric_state_dict.get("eval"), + "reg/eval", + ) + + # Per-rank cumulative metric state restore. This runs AFTER the generic + # load above so it is authoritative: the shared blob carries no lifetime + # buffers (stripped at save) nor any eval_cum state, and each rank + # restores its OWN cumulative state here. Missing file = legacy/pre-fix + # checkpoint; cumulative metrics self-heal (lifetime AUC refills; the + # binned-AUC histograms / scalar sums restart from zero). + mb_path = f"{path}/{METRICBUF_FILE_FMT.format(rank=rank)}" + if os.path.exists(mb_path): + perrank_state = torch.load( + mb_path, map_location=device, weights_only=False + ) + _restore_perrank_metric_state(metric_logger, perrank_state) + logger.info( + "checkpoint load: cumulative metric state rank=%d samples=%s", + rank, + _perrank_sample_counts(metric_logger), + ) + else: + logger.info( + "checkpoint load: no per-rank cumulative metric state at %s " + "(legacy/pre-fix checkpoint); cumulative metrics will refill", + mb_path, + ) + + # Per-rank RNG restore. Missing file = bit-equal trajectory not requested at + # save time; we silently continue (the test harness checks for both). + rng_path = f"{path}/rng_rank{rank}.pt" + if os.path.exists(rng_path): + # weights_only=False: RNG state is numpy/Python tuples, not tensors. + rng_state = torch.load(rng_path, map_location="cpu", weights_only=False) + _restore_rng_state(rng_state, device) + logger.info("RNG state restored from %s", rng_path) + + train_ts = non_sparse_state_dict.get("train_ts") + batch_idx_in_window = non_sparse_state_dict.get( + "batch_idx_in_window", WINDOW_COMPLETE + ) + split_contract = non_sparse_state_dict.get("split_contract") + return train_ts, batch_idx_in_window, split_contract + + +@gin.configurable +def load_dmp_checkpoint( + model: torch.nn.Module, + optimizer: Optimizer, + metric_logger: MetricsLogger, + device: torch.device, + path: str = "", + rank: int = 0, +) -> Tuple[Optional[int], int, Optional[Dict[str, Any]], bool]: + """ + Load a complete distributed model checkpoint (both sparse and dense components). + + `path` is auto-resolved: if it points at a directory containing numeric + subdirs (e.g. CKPT_PATH=/), the highest-numbered subdir is used. If it + already names a leaf save (e.g. /300), it's used as-is. Empty string = + no load. + + Returns: + (train_ts, batch_idx_in_window, split_contract, cold_start) — streaming + resume hint plus the saved split contract, and `cold_start` which is True + iff there was nothing to load (no checkpoint resolved). `cold_start` + distinguishes a genuine fresh run (no weights loaded) from a resume that + merely lacks a split contract (e.g. a legacy/non-streaming checkpoint), + which the caller's split-contract guard must still reject. + """ + resolved = _resolve_latest_subdir(path) + cold_start = resolved == "" + load_sparse_checkpoint(model=model, path=resolved) + train_ts, batch_idx_in_window, split_contract = load_nonsparse_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + path=resolved, + device=device, + rank=rank, + ) + return train_ts, batch_idx_in_window, split_contract, cold_start diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/configs.py b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py new file mode 100644 index 000000000..387fb4900 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/configs.py @@ -0,0 +1,830 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +Configuration module for DLRMv3 model. + +This module provides configuration functions for the HSTU model architecture and embedding table configurations. +""" + +import hashlib +import math +import os +from typing import Callable, Dict, Optional, Tuple + +import gin +import torch + +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from generative_recommenders.modules.multitask_module import ( + MultitaskTaskType, + TaskConfig, +) +from torchrec.modules.embedding_configs import DataType, EmbeddingConfig + +HSTU_EMBEDDING_DIM = 512 # final DLRMv3 model +HASH_SIZE = 10_000_000 +HASH_SIZE_1B = 1_000_000_000 + +# (name, keys, num_embeddings, salt) — single source of truth for both +# get_embedding_table_config("yambda-5b") and the dataset's cross-hash inputs. +# Sizes mirror Primus-DLRM/configs/bench_onetrans_large_5b_cross_feat_shampoo.yaml. +YAMBDA_5B_CROSS_SPECS = [ + ("user_x_artist", ("uid", "artist_id"), 100_000_000, 0), + ("user_x_album", ("uid", "album_id"), 40_000_000, 0), + ("user_x_hour", ("uid", "hour_of_day"), 24_000_000, 0), + ("item_x_hour", ("item_id", "hour_of_day"), 40_000_000, 0), + ("artist_x_hour", ("artist_id", "hour_of_day"), 32_000_000, 0), + ("user_x_is_organic", ("uid", "is_organic"), 2_000_000, 0), + ("user_x_artist_x_hour", ("uid", "artist_id", "hour_of_day"), 40_000_000, 0), +] + + +@gin.configurable +def get_hstu_configs( + dataset: str = "debug", + max_seq_len: Optional[int] = None, + max_num_candidates: Optional[int] = None, + hstu_embedding_table_dim: Optional[int] = None, + hstu_transducer_embedding_dim: Optional[int] = None, + hstu_num_heads: Optional[int] = None, + hstu_attn_num_layers: Optional[int] = None, + hstu_attn_linear_dim: Optional[int] = None, + hstu_attn_qk_dim: Optional[int] = None, + hstu_input_dropout_ratio: Optional[float] = None, + hstu_linear_dropout_rate: Optional[float] = None, +) -> DlrmHSTUConfig: + """ + Create and return HSTU model configuration. + + Builds a complete DlrmHSTUConfig with default hyperparameters for the HSTU + architecture including attention settings, embedding dimensions, dropout rates, + and feature name mappings. + + Args: + dataset: Dataset identifier (currently unused, reserved for dataset-specific configs). + + Returns: + DlrmHSTUConfig: Complete configuration object for the HSTU model. + """ + hstu_config = DlrmHSTUConfig( + hstu_num_heads=4, + hstu_attn_linear_dim=128, + hstu_attn_qk_dim=128, + hstu_attn_num_layers=5, + hstu_embedding_table_dim=HSTU_EMBEDDING_DIM, + hstu_preprocessor_hidden_dim=256, + hstu_transducer_embedding_dim=512, + hstu_group_norm=False, + hstu_input_dropout_ratio=0.2, + hstu_linear_dropout_rate=0.1, + causal_multitask_weights=0.2, + ) + if "movielens" in dataset: + assert dataset in [ + "movielens-1m", + "movielens-20m", + "movielens-13b", + "movielens-18b", + ] + hstu_config.user_embedding_feature_names = ( + [ + "movie_id", + "user_id", + "sex", + "age_group", + "occupation", + "zip_code", + ] + if dataset == "movielens-1m" + else [ + "movie_id", + "user_id", + ] + ) + hstu_config.item_embedding_feature_names = [ + "item_movie_id", + ] + hstu_config.uih_post_id_feature_name = "movie_id" + hstu_config.uih_action_time_feature_name = "action_timestamp" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.candidates_weight_feature_name = "item_action_weights" + hstu_config.uih_weight_feature_name = "item_weights" + hstu_config.candidates_watchtime_feature_name = "item_movie_rating" + hstu_config.action_weights = [1, 2, 4, 8, 16] + hstu_config.contextual_feature_to_max_length = ( + { + "user_id": 1, + "sex": 1, + "age_group": 1, + "occupation": 1, + "zip_code": 1, + } + if dataset == "movielens-1m" + else { + "user_id": 1, + } + ) + hstu_config.contextual_feature_to_min_uih_length = ( + { + "user_id": 20, + "sex": 20, + "age_group": 20, + "occupation": 20, + "zip_code": 20, + } + if dataset == "movielens-1m" + else { + "user_id": 20, + } + ) + hstu_config.merge_uih_candidate_feature_mapping = [ + ("movie_id", "item_movie_id"), + ("movie_rating", "item_movie_rating"), + ("action_timestamp", "item_query_time"), + ("item_weights", "item_action_weights"), + ("dummy_watch_time", "item_dummy_watchtime"), + ] + hstu_config.hstu_uih_feature_names = ( + [ + "user_id", + "sex", + "age_group", + "occupation", + "zip_code", + "movie_id", + "movie_rating", + "action_timestamp", + "item_weights", + "dummy_watch_time", + ] + if dataset == "movielens-1m" + else [ + "user_id", + "movie_id", + "movie_rating", + "action_timestamp", + "item_weights", + "dummy_watch_time", + ] + ) + hstu_config.hstu_candidate_feature_names = [ + "item_movie_id", + "item_movie_rating", + "item_query_time", + "item_action_weights", + "item_dummy_watchtime", + ] + hstu_config.max_num_candidates = 10 + hstu_config.max_num_candidates_inference = ( + 5 if dataset not in ["movielens-13b", "movielens-18b"] else 2048 + ) + hstu_config.multitask_configs = [ + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ) + ] + elif "streaming" in dataset: + hstu_config.user_embedding_feature_names = [ + "item_id", + "user_id", + "item_category_id", + ] + hstu_config.item_embedding_feature_names = [ + "item_candidate_id", + "item_candidate_category_id", + ] + hstu_config.uih_post_id_feature_name = "item_id" + hstu_config.uih_action_time_feature_name = "action_timestamp" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.candidates_weight_feature_name = "item_action_weights" + hstu_config.uih_weight_feature_name = "item_weights" + hstu_config.candidates_watchtime_feature_name = "item_rating" + hstu_config.action_weights = [1, 2, 4, 8, 16] + hstu_config.action_embedding_init_std = 5.0 + hstu_config.contextual_feature_to_max_length = {"user_id": 1} + hstu_config.contextual_feature_to_min_uih_length = {"user_id": 20} + hstu_config.merge_uih_candidate_feature_mapping = [ + ("item_id", "item_candidate_id"), + ("item_rating", "item_candidate_rating"), + ("action_timestamp", "item_query_time"), + ("item_weights", "item_action_weights"), + ("dummy_watch_time", "item_dummy_watchtime"), + ("item_category_id", "item_candidate_category_id"), + ] + hstu_config.hstu_uih_feature_names = [ + "user_id", + "item_id", + "item_rating", + "action_timestamp", + "item_weights", + "dummy_watch_time", + "item_category_id", + ] + hstu_config.hstu_candidate_feature_names = [ + "item_candidate_id", + "item_candidate_rating", + "item_query_time", + "item_action_weights", + "item_dummy_watchtime", + "item_candidate_category_id", + ] + hstu_config.max_num_candidates = 32 + hstu_config.max_num_candidates_inference = 2048 + hstu_config.multitask_configs = [ + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ) + ] + elif "kuairand" in dataset: + hstu_config.user_embedding_feature_names = [ + "video_id", + "user_id", + "user_active_degree", + "follow_user_num_range", + "fans_user_num_range", + "friend_user_num_range", + "register_days_range", + ] + hstu_config.item_embedding_feature_names = [ + "item_video_id", + ] + hstu_config.uih_post_id_feature_name = "video_id" + hstu_config.uih_action_time_feature_name = "action_timestamp" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.uih_weight_feature_name = "action_weight" + hstu_config.candidates_weight_feature_name = "item_action_weight" + hstu_config.candidates_watchtime_feature_name = "item_target_watchtime" + # There are more contextual features in the dataset, see https://kuairand.com/ for details + hstu_config.contextual_feature_to_max_length = { + "user_id": 1, + "user_active_degree": 1, + "follow_user_num_range": 1, + "fans_user_num_range": 1, + "friend_user_num_range": 1, + "register_days_range": 1, + } + hstu_config.merge_uih_candidate_feature_mapping = [ + ("video_id", "item_video_id"), + ("action_timestamp", "item_query_time"), + ("action_weight", "item_action_weight"), + ("watch_time", "item_target_watchtime"), + ] + hstu_config.hstu_uih_feature_names = [ + "user_id", + "user_active_degree", + "follow_user_num_range", + "fans_user_num_range", + "friend_user_num_range", + "register_days_range", + "video_id", + "action_timestamp", + "action_weight", + "watch_time", + ] + hstu_config.hstu_candidate_feature_names = [ + "item_video_id", + "item_action_weight", + "item_target_watchtime", + "item_query_time", + ] + hstu_config.multitask_configs = [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_like", + task_weight=2, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_follow", + task_weight=4, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_comment", + task_weight=8, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_forward", + task_weight=16, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_hate", + task_weight=32, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="long_view", + task_weight=64, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_profile_enter", + task_weight=128, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + ] + hstu_config.action_weights = [1, 2, 4, 8, 16, 32, 64, 128] + elif "yambda" in dataset: + assert dataset in ["yambda-5b"] + cross_names = [name for (name, _k, _n, _s) in YAMBDA_5B_CROSS_SPECS] + # Per-table dim defaults to HSTU_EMBEDDING_DIM (512); override via the + # `get_hstu_configs.hstu_embedding_table_dim = N` gin binding if needed. + # Note: the embedding tables in get_embedding_table_config also use + # HSTU_EMBEDDING_DIM and must stay aligned with this value. + hstu_config.hstu_embedding_table_dim = HSTU_EMBEDDING_DIM + hstu_config.hstu_transducer_embedding_dim = 512 + hstu_config.max_seq_len = 8192 + hstu_config.max_num_candidates = 1 + hstu_config.max_num_candidates_inference = 1 + # Per dlrm_hstu convention (see streaming-100b/movielens): + # - user_embedding_feature_names = UIH-side post-id features + contextual features. + # After main_forward merges UIH + candidate, only these entries hold the merged + # sequence (used by user-side transducer). + # - item_embedding_feature_names = candidate-side names only. _item_forward + # concats these along dim=-1 to feed the item MLP (per-candidate, not per-position). + hstu_config.user_embedding_feature_names = ( + ["uid"] + + cross_names + + ["item_id", "artist_id", "album_id"] + ) + hstu_config.item_embedding_feature_names = [ + "item_candidate_id", + "item_candidate_artist_id", + "item_candidate_album_id", + ] + hstu_config.uih_post_id_feature_name = "item_id" + hstu_config.uih_action_time_feature_name = "action_timestamp" + hstu_config.uih_weight_feature_name = "action_weight" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.candidates_weight_feature_name = "item_action_weight" + hstu_config.candidates_watchtime_feature_name = "item_dummy_watchtime" + hstu_config.action_weights = [1, 2, 4] # lp, like, skip bits + hstu_config.contextual_feature_to_max_length = { + "uid": 1, + **{name: 1 for name in cross_names}, + } + hstu_config.contextual_feature_to_min_uih_length = { + "uid": 0, + **{name: 0 for name in cross_names}, + } + # uih names map to candidate names (no name collisions allowed): + # item_id/artist_id/album_id appear with prefix "item_" on candidate side. + hstu_config.merge_uih_candidate_feature_mapping = [ + ("item_id", "item_candidate_id"), + ("artist_id", "item_candidate_artist_id"), + ("album_id", "item_candidate_album_id"), + ("action_weight", "item_action_weight"), + ("action_timestamp", "item_query_time"), + ("dummy_watch_time", "item_dummy_watchtime"), + ] + hstu_config.hstu_uih_feature_names = ( + ["uid"] + + cross_names + + [ + "item_id", + "artist_id", + "album_id", + "action_weight", + "action_timestamp", + "dummy_watch_time", + ] + ) + hstu_config.hstu_candidate_feature_names = [ + "item_candidate_id", + "item_candidate_artist_id", + "item_candidate_album_id", + "item_query_time", + "item_action_weight", + "item_dummy_watchtime", + ] + hstu_config.multitask_configs = [ + TaskConfig( + task_name="listen_plus", + task_weight=1, # matches action_weights[0] (lp bit) + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ) + ] + else: + hstu_config.user_embedding_feature_names = [ + "uih_post_id", + "uih_owner_id", + "viewer_id", + "dummy_contexual", + ] + hstu_config.item_embedding_feature_names = [ + "item_post_id", + "item_owner_id", + ] + hstu_config.uih_post_id_feature_name = "uih_post_id" + hstu_config.uih_action_time_feature_name = "uih_action_time" + hstu_config.candidates_querytime_feature_name = "item_query_time" + hstu_config.candidates_weight_feature_name = "item_action_weight" + hstu_config.candidates_watchtime_feature_name = "item_target_watchtime" + hstu_config.contextual_feature_to_max_length = { + "viewer_id": 1, + "dummy_contexual": 1, + } + hstu_config.contextual_feature_to_min_uih_length = { + "viewer_id": 128, + "dummy_contexual": 128, + } + hstu_config.merge_uih_candidate_feature_mapping = [ + ("uih_post_id", "item_post_id"), + ("uih_owner_id", "item_owner_id"), + ("uih_action_time", "item_query_time"), + ("uih_weight", "item_action_weight"), + ("uih_watchtime", "item_target_watchtime"), + ("uih_video_length", "item_video_length"), + ("uih_surface_type", "item_surface_type"), + ] + hstu_config.hstu_uih_feature_names = [ + "uih_post_id", + "uih_action_time", + "uih_weight", + "uih_owner_id", + "uih_watchtime", + "uih_surface_type", + "uih_video_length", + "viewer_id", + "dummy_contexual", + ] + hstu_config.hstu_candidate_feature_names = [ + "item_post_id", + "item_owner_id", + "item_surface_type", + "item_video_length", + "item_action_weight", + "item_target_watchtime", + "item_query_time", + ] + hstu_config.multitask_configs = [ + TaskConfig( + task_name="vvp100", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ) + ] + + # Apply gin overrides last so a value set in the gin file wins over the + # per-dataset defaults above. Anything left as None inherits the default + # the dataset branch (or DlrmHSTUConfig) chose. Example in a gin file: + # get_hstu_configs.max_seq_len = 4096 + # get_hstu_configs.hstu_embedding_table_dim = 256 + _gin_overrides = { + "max_seq_len": max_seq_len, + "max_num_candidates": max_num_candidates, + "max_num_candidates_inference": max_num_candidates, + "hstu_embedding_table_dim": hstu_embedding_table_dim, + "hstu_transducer_embedding_dim": hstu_transducer_embedding_dim, + "hstu_num_heads": hstu_num_heads, + "hstu_attn_num_layers": hstu_attn_num_layers, + "hstu_attn_linear_dim": hstu_attn_linear_dim, + "hstu_attn_qk_dim": hstu_attn_qk_dim, + "hstu_input_dropout_ratio": hstu_input_dropout_ratio, + "hstu_linear_dropout_rate": hstu_linear_dropout_rate, + } + for _name, _val in _gin_overrides.items(): + if _val is not None: + setattr(hstu_config, _name, _val) + + return hstu_config + + +def _stable_table_seed(init_seed: int, table_name: str) -> int: + """Deterministic 63-bit seed from (init_seed, table_name). + + Uses sha256 (not Python's salted built-in ``hash()``) so the per-table seed + is identical across processes/ranks/runs for a given ``$SEED`` + table name. + """ + digest = hashlib.sha256(f"{init_seed}:{table_name}".encode("utf-8")).digest() + return int.from_bytes(digest[:8], "big") & 0x7FFF_FFFF_FFFF_FFFF + + +def _uniform_init_bounds(cfg: EmbeddingConfig) -> Tuple[float, float]: + """Mirror TorchREC's default per-table init bounds. + + TorchREC falls back to ``uniform_(-1/sqrt(N), +1/sqrt(N))`` when a table does + not set ``weight_init_min/max``; honor any explicit bounds the config carries. + """ + bound = math.sqrt(1.0 / cfg.num_embeddings) + lo = -bound if cfg.weight_init_min is None else cfg.weight_init_min + hi = bound if cfg.weight_init_max is None else cfg.weight_init_max + return lo, hi + + +def _make_seeded_uniform_init( + table_seed: int, lo: float, hi: float +) -> Callable[[torch.Tensor], torch.Tensor]: + """Build a seeded in-place uniform initializer for one table's weight. + + TorchREC/FBGEMM calls ``init_fn`` with the (per-rank) local shard tensor on + its compute device, so we seed a generator on that same device. For a fixed + sharding plan (world size + plan unchanged) this makes embedding init + byte-reproducible run-to-run. + """ + + def _init(weight: torch.Tensor) -> torch.Tensor: + # TorchREC builds the unsharded EmbeddingCollection on the META device + # first (DMP materializes real storage on the compute device later). + # Meta tensors have no storage and torch.Generator(device="meta") is + # invalid ("META device type not an accelerator"), so skip them: the + # seeded init for the sharded/fused TBE path is provided by the RNG + # re-seed right before DMP in make_optimizer_and_shard. On a real + # device (eager/non-meta path) we still apply the per-table seeded fill. + if weight.device.type == "meta": + return weight + gen = torch.Generator(device=weight.device) + gen.manual_seed(table_seed) + with torch.no_grad(): + weight.uniform_(lo, hi, generator=gen) + return weight + + return _init + + +@gin.configurable +def get_embedding_table_config( + dataset: str = "debug", + embedding_dim: Optional[int] = None, + init_seed: Optional[int] = None, +) -> Dict[str, EmbeddingConfig]: + """ + Create and return embedding table configurations. + + Defines the embedding table configurations for item IDs, category IDs, and user IDs + with their respective dimensions and data types. + + Args: + dataset: Dataset identifier (currently unused, reserved for dataset-specific configs). + embedding_dim: Per-table embedding width override. When set via gin + (e.g. `get_embedding_table_config.embedding_dim = 256`), wins over + `HSTU_EMBEDDING_DIM`. Keep in sync with the matching gin override on + `get_hstu_configs.hstu_embedding_table_dim` — the model and the + tables must agree on dim or sharding will reject the plan. + init_seed: Base seed for the per-table seeded `init_fn` (Tier 1 + reproducible embedding init). When None, falls back to `$SEED` + (default 1), matching `seed_everything`. Each table draws from a + generator seeded by `sha256(init_seed, table_name)` so init is + reproducible run-to-run for a fixed sharding plan. + + Returns: + Dict mapping table names to their EmbeddingConfig objects. + """ + tables = _build_embedding_table_config(dataset=dataset, embedding_dim=embedding_dim) + + if init_seed is None: + init_seed = int(os.environ.get("SEED", "1")) + for name, cfg in tables.items(): + lo, hi = _uniform_init_bounds(cfg) + cfg.init_fn = _make_seeded_uniform_init( + _stable_table_seed(init_seed, name), lo, hi + ) + return tables + + +def _build_embedding_table_config( + dataset: str = "debug", + embedding_dim: Optional[int] = None, +) -> Dict[str, EmbeddingConfig]: + DIM = embedding_dim if embedding_dim is not None else HSTU_EMBEDDING_DIM + if "movielens" in dataset: + assert dataset in [ + "movielens-1m", + "movielens-20m", + "movielens-13b", + "movielens-18b", + ] + return ( + { + "movie_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="movie_id", + data_type=DataType.FP16, + feature_names=["movie_id", "item_movie_id"], + ), + "user_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="user_id", + data_type=DataType.FP16, + feature_names=["user_id"], + ), + "sex": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="sex", + data_type=DataType.FP16, + feature_names=["sex"], + ), + "age_group": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="age_group", + data_type=DataType.FP16, + feature_names=["age_group"], + ), + "occupation": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="occupation", + data_type=DataType.FP16, + feature_names=["occupation"], + ), + "zip_code": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="zip_code", + data_type=DataType.FP16, + feature_names=["zip_code"], + ), + } + if dataset == "movielens-1m" + else { + "movie_id": EmbeddingConfig( + num_embeddings=HASH_SIZE_1B, + embedding_dim=DIM, + name="movie_id", + data_type=DataType.FP16, + feature_names=["movie_id", "item_movie_id"], + ), + "user_id": EmbeddingConfig( + num_embeddings=3_000_000, + embedding_dim=DIM, + name="user_id", + data_type=DataType.FP16, + feature_names=["user_id"], + ), + } + ) + elif "streaming" in dataset: + return { + "item_id": EmbeddingConfig( + num_embeddings=HASH_SIZE_1B, + embedding_dim=DIM, + name="item_id", + data_type=DataType.FP16, + feature_names=["item_id", "item_candidate_id"], + ), + "item_category_id": EmbeddingConfig( + num_embeddings=128, + embedding_dim=DIM, + name="item_category_id", + data_type=DataType.FP16, + weight_init_max=1.0, + weight_init_min=-1.0, + feature_names=["item_category_id", "item_candidate_category_id"], + ), + "user_id": EmbeddingConfig( + num_embeddings=10_000_000, + embedding_dim=DIM, + name="user_id", + data_type=DataType.FP16, + feature_names=["user_id"], + ), + } + elif "kuairand" in dataset: + return { + "video_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="video_id", + data_type=DataType.FP16, + feature_names=["video_id", "item_video_id"], + ), + "user_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="user_id", + data_type=DataType.FP16, + feature_names=["user_id"], + ), + "user_active_degree": EmbeddingConfig( + num_embeddings=8, + embedding_dim=DIM, + name="user_active_degree", + data_type=DataType.FP16, + feature_names=["user_active_degree"], + ), + "follow_user_num_range": EmbeddingConfig( + num_embeddings=9, + embedding_dim=DIM, + name="follow_user_num_range", + data_type=DataType.FP16, + feature_names=["follow_user_num_range"], + ), + "fans_user_num_range": EmbeddingConfig( + num_embeddings=9, + embedding_dim=DIM, + name="fans_user_num_range", + data_type=DataType.FP16, + feature_names=["fans_user_num_range"], + ), + "friend_user_num_range": EmbeddingConfig( + num_embeddings=8, + embedding_dim=DIM, + name="friend_user_num_range", + data_type=DataType.FP16, + feature_names=["friend_user_num_range"], + ), + "register_days_range": EmbeddingConfig( + num_embeddings=8, + embedding_dim=DIM, + name="register_days_range", + data_type=DataType.FP16, + feature_names=["register_days_range"], + ), + } + elif "yambda" in dataset: + assert dataset in ["yambda-5b"] + tables: Dict[str, EmbeddingConfig] = { + "item_id": EmbeddingConfig( + num_embeddings=9_390_624, + embedding_dim=DIM, + name="item_id", + data_type=DataType.FP32, + feature_names=["item_id", "item_candidate_id"], + ), + "artist_id": EmbeddingConfig( + num_embeddings=1_293_395, + embedding_dim=DIM, + name="artist_id", + data_type=DataType.FP32, + feature_names=["artist_id", "item_candidate_artist_id"], + ), + "album_id": EmbeddingConfig( + num_embeddings=3_367_692, + embedding_dim=DIM, + name="album_id", + data_type=DataType.FP32, + feature_names=["album_id", "item_candidate_album_id"], + ), + "uid": EmbeddingConfig( + num_embeddings=1_000_001, + embedding_dim=DIM, + name="uid", + data_type=DataType.FP32, + feature_names=["uid"], + ), + } + for name, _keys, num_embeddings, _salt in YAMBDA_5B_CROSS_SPECS: + tables[name] = EmbeddingConfig( + num_embeddings=num_embeddings, + embedding_dim=DIM, + name=name, + data_type=DataType.FP32, + feature_names=[name], + ) + return tables + else: + return { + "post_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="post_id", + data_type=DataType.FP16, + feature_names=[ + "uih_post_id", + "item_post_id", + "uih_owner_id", + "item_owner_id", + ], + ), + "viewer_id": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="viewer_id", + data_type=DataType.FP16, + feature_names=["viewer_id"], + ), + "dummy_contexual": EmbeddingConfig( + num_embeddings=HASH_SIZE, + embedding_dim=DIM, + name="dummy_contexual", + data_type=DataType.FP16, + feature_names=["dummy_contexual"], + ), + } diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py new file mode 100644 index 000000000..204c06df1 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/dataset.py @@ -0,0 +1,461 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +""" +Dataset implementations for DLRMv3. + +This module provides dataset classes for loading and processing recommendation +data, including sample containers, collation functions, and random data generation. +""" + +import logging +import time +from dataclasses import dataclass +from typing import Dict, List, Tuple + +import torch +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor +from torchrec.streamable import Pipelineable + + +logging.basicConfig(level=logging.INFO) +logger: logging.Logger = logging.getLogger("dlrmv3_dataset") + + +@dataclass +class Samples(Pipelineable): + """ + Container for batched samples with user interaction history and candidate features. + + Attributes: + uih_features_kjt: User interaction history features as KeyedJaggedTensor. + candidates_features_kjt: Candidate item features as KeyedJaggedTensor. + """ + + uih_features_kjt: KeyedJaggedTensor + candidates_features_kjt: KeyedJaggedTensor + # UIH + candidate features concatenated into the single KJT that the model's + # sharded EmbeddingCollection consumes. Pre-built here (dataloader/CPU) rather + # than inside DlrmHSTU.forward so the embedding lookup's input is a plain + # attribute of the batch — which lets TorchRec's TrainPipelineSparseDist hoist + # its input_dist into the prefetch stage (otherwise the runtime cat + + # from_lengths_sync counts as an "input modification" and the embedding + # collection is left un-pipelined). + merged_sparse_features: KeyedJaggedTensor + + def to(self, device: torch.device, non_blocking: bool = False) -> "Samples": + """ + Move all tensors to the specified device (in place) and return self. + + Returning ``self`` (rather than ``None``) and accepting ``non_blocking`` + makes ``Samples`` conform to TorchRec's ``Pipelineable`` protocol so it + can be driven by ``TrainPipelineSparseDist``. Existing call sites that + use ``sample.to(device)`` for its side effect continue to work unchanged. + """ + for attr in vars(self): + setattr( + self, + attr, + getattr(self, attr).to(device=device, non_blocking=non_blocking), + ) + return self + + def record_stream(self, stream: torch.Stream) -> None: + """Record the contained KJTs on ``stream`` (Pipelineable protocol). + + Required by ``TrainPipelineSparseDist`` so the prefetched batch's H2D + copy on the side stream is not freed before compute consumes it. + """ + self.uih_features_kjt.record_stream(stream) + self.candidates_features_kjt.record_stream(stream) + self.merged_sparse_features.record_stream(stream) + + def pin_memory(self) -> "Samples": + """Pin the contained KJTs' host memory (Pipelineable protocol).""" + self.uih_features_kjt = self.uih_features_kjt.pin_memory() + self.candidates_features_kjt = self.candidates_features_kjt.pin_memory() + self.merged_sparse_features = self.merged_sparse_features.pin_memory() + return self + + def batch_size(self) -> int: + """ + Get the batch size of the samples. + + Returns: + Number of samples in the batch. + """ + return self.uih_features_kjt.stride() + + +def merge_uih_candidate_kjts( + uih_features: KeyedJaggedTensor, + candidates_features: KeyedJaggedTensor, +) -> KeyedJaggedTensor: + """Concatenate the UIH and candidate KJTs into the single KJT consumed by the + model's ``EmbeddingCollection``. + + Must mirror ``DlrmHSTU.preprocess`` exactly (key order = uih + candidates, + values/lengths concatenated in that order). Built on the dataloader side so + the model can read it straight off the batch and TorchRec can pipeline the + embedding ``input_dist``. + """ + return KeyedJaggedTensor.from_lengths_sync( + keys=uih_features.keys() + candidates_features.keys(), + values=torch.cat( + [uih_features.values(), candidates_features.values()], + dim=0, + ), + lengths=torch.cat( + [uih_features.lengths(), candidates_features.lengths()], + dim=0, + ), + ) + + +def collate_fn( + samples: List[Tuple[KeyedJaggedTensor, KeyedJaggedTensor]], +) -> Samples: + """ + Collate multiple samples into a batched Samples object. + + Args: + samples: List of (uih_features, candidates_features) tuples. + + Returns: + Batched Samples object with concatenated features. + """ + ( + uih_features_kjt_list, + candidates_features_kjt_list, + ) = list(zip(*samples)) + + uih_features_kjt = kjt_batch_func(uih_features_kjt_list) + candidates_features_kjt = kjt_batch_func(candidates_features_kjt_list) + return Samples( + uih_features_kjt=uih_features_kjt, + candidates_features_kjt=candidates_features_kjt, + merged_sparse_features=merge_uih_candidate_kjts( + uih_features_kjt, candidates_features_kjt + ), + ) + + +class Dataset: + """ + Base dataset class for DLRMv3. + + Provides the interface for loading, accessing, and managing samples + for recommendation model training and inference. + + Args: + hstu_config: HSTU model configuration. + **args: Additional arguments (unused in base class). + """ + + def __init__(self, hstu_config: DlrmHSTUConfig, **args): + self.arrival = None + self.image_list = [] + self.label_list = [] + self.image_list_inmemory = {} + self.last_loaded = -1.0 + + def preprocess(self, use_cache=True): + """ + Preprocess the dataset. + + Args: + use_cache: Whether to use cached preprocessed data. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("Dataset:preprocess") + + def get_item_count(self): + """ + Get the total number of items in the dataset. + + Returns: + Number of items. + """ + return len(self.image_list) + + def load_query_samples(self, sample_list): + """ + Load specified samples into memory. + + Args: + sample_list: List of sample indices to load. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("Dataset:load_query_samples") + + def unload_query_samples(self, sample_list): + """ + Unload specified samples from memory. + + Args: + sample_list: List of sample indices to unload. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("Dataset:unload_query_samples") + + def get_sample(self, id: int): + """ + Get a single sample by ID. + + Args: + id: Sample identifier. + + Raises: + NotImplementedError: Subclasses must implement this method. + """ + raise NotImplementedError("Dataset:get_sample") + + def get_samples(self, id_list: List[int]) -> Samples: + """ + Get multiple samples and collate them into a batch. + + Args: + id_list: List of sample identifiers. + + Returns: + Collated Samples object containing the batch. + """ + list_samples = [self.get_sample(ix) for ix in id_list] + return collate_fn(list_samples) + + +@torch.jit.script +def kjt_batch_func( + kjt_list: List[KeyedJaggedTensor], +) -> KeyedJaggedTensor: + """ + Batch multiple KeyedJaggedTensors into a single tensor. + + Uses FBGEMM operations for efficient batching and reordering of + jagged tensor data. + + Args: + kjt_list: List of KeyedJaggedTensors to batch. + + Returns: + Batched KeyedJaggedTensor with reordered indices and lengths. + """ + bs_list = [kjt.stride() for kjt in kjt_list] + bs = sum(bs_list) + batched_length = torch.cat([kjt.lengths() for kjt in kjt_list], dim=0) + batched_indices = torch.cat([kjt.values() for kjt in kjt_list], dim=0) + bs_offset = torch.ops.fbgemm.asynchronous_complete_cumsum( + torch.tensor(bs_list) + ).int() + batched_offset = torch.ops.fbgemm.asynchronous_complete_cumsum(batched_length) + reorder_length = torch.ops.fbgemm.reorder_batched_ad_lengths( + batched_length, bs_offset, bs + ) + reorder_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(reorder_length) + reorder_indices = torch.ops.fbgemm.reorder_batched_ad_indices( + batched_offset, batched_indices, reorder_offsets, bs_offset, bs + ) + out = KeyedJaggedTensor( + keys=kjt_list[0].keys(), + lengths=reorder_length.long(), + values=reorder_indices.long(), + ) + return out + + +def get_random_data( + contexual_features: List[str], + hstu_uih_keys: List[str], + hstu_candidates_keys: List[str], + uih_max_seq_len: int, + max_num_candidates: int, + value_bound: int = 1000, +): + """ + Generate random sample data for testing and debugging. + + Creates synthetic user interaction history and candidate features + with random values. + + Args: + contexual_features: List of contextual feature names. + hstu_uih_keys: List of UIH feature keys. + hstu_candidates_keys: List of candidate feature keys. + uih_max_seq_len: Maximum sequence length for UIH. + max_num_candidates: Maximum number of candidates. + value_bound: Upper bound for random values. + + Returns: + Tuple of (uih_features_kjt, candidates_features_kjt). + """ + uih_non_seq_feature_keys = contexual_features + uih_seq_feature_keys = [ + k for k in hstu_uih_keys if k not in uih_non_seq_feature_keys + ] + uih_seq_len = torch.randint( + int(uih_max_seq_len * 0.8), + uih_max_seq_len + 1, + (1,), + ).item() + uih_lengths = torch.tensor( + [1 for _ in uih_non_seq_feature_keys] + + [uih_seq_len for _ in uih_seq_feature_keys] + ) + # logging.info(f"uih_lengths: {uih_lengths}") + uih_values = torch.randint( + 1, + value_bound, + # pyre-ignore[6] + (uih_seq_len * len(uih_seq_feature_keys) + len(uih_non_seq_feature_keys),), + ) + uih_features_kjt = KeyedJaggedTensor( + keys=uih_non_seq_feature_keys + uih_seq_feature_keys, + lengths=uih_lengths.long(), + values=uih_values.long(), + ) + num_candidates = torch.randint( + 1, + max_num_candidates + 1, + (1,), + ).item() + candidates_lengths = num_candidates * torch.ones(len(hstu_candidates_keys)) + candidates_values = torch.randint( + 1, + value_bound, + (num_candidates * len(hstu_candidates_keys),), # pyre-ignore[6] + ) + candidates_features_kjt = KeyedJaggedTensor( + keys=hstu_candidates_keys, + lengths=candidates_lengths.long(), + values=candidates_values.long(), + ) + return uih_features_kjt, candidates_features_kjt + + +class DLRMv3RandomDataset(Dataset): + """ + Dataset that generates random synthetic data for DLRMv3. + + Useful for testing and benchmarking without real data dependencies. + + Args: + hstu_config: HSTU model configuration. + num_aggregated_samples: Total number of samples to generate. + is_inference: Whether the dataset is used for inference mode. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + num_aggregated_samples: int = 10000, + is_inference: bool = False, + *args, + **kwargs, + ): + super().__init__( + hstu_config=hstu_config, + ) + self.hstu_config: DlrmHSTUConfig = hstu_config + self._max_num_candidates: int = hstu_config.max_num_candidates + self._max_num_candidates_inference: int = ( + hstu_config.max_num_candidates_inference + ) + self._max_seq_len: int = hstu_config.max_seq_len + self._uih_keys: List[str] = hstu_config.hstu_uih_feature_names + self._candidates_keys: List[str] = hstu_config.hstu_candidate_feature_names + self._contextual_feature_to_max_length: Dict[str, int] = ( + hstu_config.contextual_feature_to_max_length + ) + self._max_uih_len: int = ( + self._max_seq_len + - self._max_num_candidates + - ( + len(self._contextual_feature_to_max_length) + if self._contextual_feature_to_max_length + else 0 + ) + ) + self._is_inference = is_inference + + self.contexual_features = [] + if hstu_config.contextual_feature_to_max_length is not None: + self.contexual_features = [ + p[0] for p in hstu_config.contextual_feature_to_max_length + ] + + self.num_aggregated_samples = num_aggregated_samples + self.items_in_memory = {} + + def get_sample(self, id: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + """ + Get a sample by ID from in-memory storage. + + Args: + id: Sample identifier. + + Returns: + Tuple of (uih_features_kjt, candidates_features_kjt). + """ + return self.items_in_memory[id] + + def get_item_count(self): + """ + Get the total number of samples in the dataset. + + Returns: + Number of aggregated samples. + """ + return self.num_aggregated_samples + + def unload_query_samples(self, sample_list): + """ + Clear all samples from memory. + + Args: + sample_list: Ignored; clears all samples. + """ + self.items_in_memory = {} + + def load_query_samples(self, sample_list): + """ + Generate and load random samples into memory. + + Args: + sample_list: List of sample IDs to generate. + """ + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + self.items_in_memory = {} + for sample in sample_list: + self.items_in_memory[sample] = get_random_data( + contexual_features=self.contexual_features, + hstu_uih_keys=self.hstu_config.hstu_uih_feature_names, + hstu_candidates_keys=self.hstu_config.hstu_candidate_feature_names, + uih_max_seq_len=self._max_uih_len, + max_num_candidates=max_num_candidates, + ) + self.last_loaded = time.time() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/kuairand.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/kuairand.py new file mode 100644 index 000000000..f6cd9e672 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/kuairand.py @@ -0,0 +1,163 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +import json +import time +from functools import partial +from typing import Any, Dict, List + +import pandas as pd +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import DLRMv3RandomDataset +from generative_recommenders.dlrm_v3.datasets.utils import ( + maybe_truncate_seq, + separate_uih_candidates, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + + +def process_and_hash_x(x: Any, hash_size: int) -> Any: + if isinstance(x, str): + x = json.loads(x) + if isinstance(x, list): + return [x_i % hash_size for x_i in x] + else: + return x % hash_size + + +class DLRMv3KuaiRandDataset(DLRMv3RandomDataset): + def __init__( + self, + hstu_config: DlrmHSTUConfig, + embedding_config: Dict[str, Any], + seq_logs_file: str, + is_inference: bool, + **kwargs, + ) -> None: + super().__init__(hstu_config=hstu_config, is_inference=is_inference) + self.seq_logs_frame: pd.DataFrame = pd.read_csv(seq_logs_file, delimiter=",") + # apply hashing from embedding table config + for key, table in embedding_config.items(): + assert key in self.seq_logs_frame.columns, ( + "Rename key in embedding table configs!" + ) + hash_size = table.num_embeddings + self.seq_logs_frame[key] = self.seq_logs_frame[key].apply( + partial(process_and_hash_x, hash_size=hash_size) + ) + + def get_item_count(self): + return len(self.seq_logs_frame) + + def unload_query_samples(self, sample_list): + self.items_in_memory = {} + + def load_query_samples(self, sample_list): + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + self.items_in_memory = {} + for idx in sample_list: + data = self.seq_logs_frame.iloc[idx] + if len(data.video_id) <= max_num_candidates: + continue + sample = self.load_item(data, max_num_candidates) + self.items_in_memory[idx] = sample + + self.last_loaded = time.time() + + def load_item(self, data, max_num_candidates): + with torch.profiler.record_function("load_item"): + video_history_uih, video_history_candidates = separate_uih_candidates( + data.video_id, + candidates_max_seq_len=max_num_candidates, + ) + action_weights_uih, action_weights_candidates = separate_uih_candidates( + data.action_weights, + candidates_max_seq_len=max_num_candidates, + ) + timestamps_uih, _ = separate_uih_candidates( + data.time_ms, + candidates_max_seq_len=max_num_candidates, + ) + watch_time_uih, watch_time_candidates = separate_uih_candidates( + data.play_time_ms, + candidates_max_seq_len=max_num_candidates, + ) + + video_history_uih = maybe_truncate_seq(video_history_uih, self._max_uih_len) + action_weights_uih = maybe_truncate_seq( + action_weights_uih, self._max_uih_len + ) + timestamps_uih = maybe_truncate_seq(timestamps_uih, self._max_uih_len) + watch_time_uih = maybe_truncate_seq(watch_time_uih, self._max_uih_len) + + uih_seq_len = len(video_history_uih) + assert uih_seq_len == len(timestamps_uih), ( + "history len differs from timestamp len." + ) + assert uih_seq_len == len(action_weights_uih), ( + "history len differs from weights len." + ) + assert uih_seq_len == len(watch_time_uih), ( + "history len differs from watch time len." + ) + + uih_kjt_values: List[torch.Tensor] = [] + uih_kjt_lengths: List[torch.Tensor] = [] + for name, length in self._contextual_feature_to_max_length.items(): + uih_kjt_values.append(data[name]) + uih_kjt_lengths.append(length) + + uih_kjt_values.extend( + video_history_uih + timestamps_uih + action_weights_uih + watch_time_uih + ) + + uih_kjt_lengths.extend( + [ + uih_seq_len + for _ in range( + len(self._uih_keys) + - len(self._contextual_feature_to_max_length) + ) + ] + ) + + dummy_query_time = max(timestamps_uih) + uih_features_kjt = KeyedJaggedTensor( + keys=self._uih_keys, + lengths=torch.tensor(uih_kjt_lengths).long(), + values=torch.tensor(uih_kjt_values).long(), + ) + + candidates_kjt_lengths = max_num_candidates * torch.ones( + len(self._candidates_keys) + ) + candidates_kjt_values = ( + video_history_candidates + + action_weights_candidates + + watch_time_candidates + + [dummy_query_time] * max_num_candidates + ) + candidates_features_kjt = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=torch.tensor(candidates_kjt_lengths).long(), + values=torch.tensor(candidates_kjt_values).long(), + ) + + return uih_features_kjt, candidates_features_kjt diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/movie_lens.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/movie_lens.py new file mode 100644 index 000000000..d74fb575b --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/movie_lens.py @@ -0,0 +1,177 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +import logging +import time +from typing import List, Optional + +import pandas as pd +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import DLRMv3RandomDataset +from generative_recommenders.dlrm_v3.datasets.utils import ( + maybe_truncate_seq, + separate_uih_candidates, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + +logger = logging.getLogger(__name__) + + +class DLRMv3MovieLensDataset(DLRMv3RandomDataset): + def __init__( + self, + hstu_config: DlrmHSTUConfig, + ratings_file: str, + is_inference: bool, + *args, + **kwargs, + ): + super().__init__(hstu_config=hstu_config, is_inference=is_inference) + self.ratings_frame: Optional[pd.DataFrame] = None + if ratings_file != "": + self.ratings_frame = pd.read_csv( + ratings_file, + delimiter=",", + ) + assert hstu_config.action_weights is not None + self.action_weights: List[int] = hstu_config.action_weights + + def get_item_count(self): + assert self.ratings_frame is not None + return len(self.ratings_frame) + + def unload_query_samples(self, sample_list): + self.items_in_memory = {} + + def iloc(self, idx): + assert self.ratings_frame is not None + return self.ratings_frame.iloc[idx] + + def load_query_samples(self, sample_list): + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + self.items_in_memory = {} + for idx in sample_list: + data = self.iloc(idx) + if len(data.sequence_item_ids) <= max_num_candidates: + continue + sample = self.load_item(data, max_num_candidates) + self.items_in_memory[idx] = sample + + self.last_loaded = time.time() + + def get_timestamp_uih(self, data, max_num_candidates, size): + movie_timestamps_uih, _ = separate_uih_candidates( + data.sequence_timestamps, + candidates_max_seq_len=max_num_candidates, + ) + return movie_timestamps_uih + + def load_item(self, data, max_num_candidates): + movie_history_uih, movie_history_candidates = separate_uih_candidates( + data.sequence_item_ids, + candidates_max_seq_len=max_num_candidates, + ) + movie_history_ratings_uih, movie_history_ratings_candidates = ( + separate_uih_candidates( + data.sequence_ratings, + candidates_max_seq_len=max_num_candidates, + ) + ) + movie_timestamps_uih = self.get_timestamp_uih( + data=data, + max_num_candidates=max_num_candidates, + size=len(movie_history_uih), + ) + + assert len(movie_history_uih) == len(movie_timestamps_uih), ( + "history len differs from timestamp len." + ) + assert len(movie_history_uih) == len(movie_history_ratings_uih), ( + "history len differs from ratings len." + ) + + movie_history_uih = maybe_truncate_seq(movie_history_uih, self._max_uih_len) + movie_history_ratings_uih = maybe_truncate_seq( + movie_history_ratings_uih, self._max_uih_len + ) + movie_timestamps_uih = maybe_truncate_seq( + movie_timestamps_uih, self._max_uih_len + ) + + uih_kjt_values: List[torch.Tensor] = [] + uih_kjt_lengths: List[torch.Tensor] = [] + for name, length in self._contextual_feature_to_max_length.items(): + uih_kjt_values.append(data[name]) + uih_kjt_lengths.append(length) + + uih_seq_len = len(movie_history_uih) + movie_dummy_watch_times_uih = [0 for _ in range(uih_seq_len)] + action_weights_uih = [ + self.action_weights[int(rating) - 1] for rating in movie_history_ratings_uih + ] + uih_kjt_values.extend( + movie_history_uih + + movie_history_ratings_uih + + movie_timestamps_uih + + action_weights_uih + + movie_dummy_watch_times_uih + ) + uih_kjt_lengths.extend( + [ + uih_seq_len + for _ in range( + len(self._uih_keys) - len(self._contextual_feature_to_max_length) + ) + ] + ) + + dummy_query_time = ( + 0 if movie_timestamps_uih == [] else max(movie_timestamps_uih) + ) + uih_kjt_values.append(dummy_query_time) + uih_kjt_lengths.append(1) + uih_features_kjt = KeyedJaggedTensor( + keys=self._uih_keys + ["dummy_query_time"], + lengths=torch.tensor(uih_kjt_lengths).long(), + values=torch.tensor(uih_kjt_values).long(), + ) + + candidates_kjt_lengths = max_num_candidates * torch.ones( + len(self._candidates_keys) + ) + action_weights_candidates = [ + int(rating >= 3.5) for rating in movie_history_ratings_candidates + ] + candidates_kjt_values = ( + movie_history_candidates + + movie_history_ratings_candidates + + [dummy_query_time] * max_num_candidates # item_query_time + + action_weights_candidates + + [1] * max_num_candidates # item_dummy_watchtime + ) + candidates_features_kjt = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=candidates_kjt_lengths.detach().clone().long(), + values=torch.tensor(candidates_kjt_values).long(), + ) + return ( + uih_features_kjt, + candidates_features_kjt, + ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_movie_lens.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_movie_lens.py new file mode 100644 index 000000000..6cf8a5f56 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_movie_lens.py @@ -0,0 +1,83 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +import csv +import linecache +import logging +import sys +from typing import List + +import numpy as np +import pandas as pd +from generative_recommenders.dlrm_v3.datasets.movie_lens import DLRMv3MovieLensDataset +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig + +csv.field_size_limit(sys.maxsize) +logger = logging.getLogger(__name__) + + +class DLRMv3SyntheticMovieLensDataset(DLRMv3MovieLensDataset): + def __init__( + self, + hstu_config: DlrmHSTUConfig, + ratings_file_prefix: str, + is_inference: bool, + *args, + **kwargs, + ): + super().__init__( + hstu_config=hstu_config, is_inference=is_inference, ratings_file="" + ) + self.ratings_file_prefix = ratings_file_prefix + with open(f"{self.ratings_file_prefix}_users.csv", "r") as file: + reader = csv.reader(file) + self.users_cumsum: List[int] = np.cumsum( + [int(row[1]) for row in reader] + ).tolist() + + def get_item_count(self): + return self.users_cumsum[-1] + + def _process_line(self, line: str) -> pd.Series: + reader = csv.reader([line]) + parsed_line = next(reader) + user_id = int(parsed_line[0]) + sequence_item_ids = parsed_line[1] + sequence_ratings = parsed_line[2] + return pd.Series( + data={ + "user_id": user_id, + "sequence_item_ids": sequence_item_ids, + "sequence_ratings": sequence_ratings, + } + ) + + def iloc(self, idx) -> pd.Series: + assert idx < self.users_cumsum[-1] + file_idx: int = 0 + while self.users_cumsum[file_idx] <= idx: + file_idx += 1 + if file_idx == 0: + local_idx = idx + else: + local_idx = idx - self.users_cumsum[file_idx - 1] + line = linecache.getline( + f"{self.ratings_file_prefix}_{file_idx}.csv", local_idx + 1 + ) + data = self._process_line(line) + return data + + def get_timestamp_uih(self, data, max_num_candidates, size): + return [1] * size diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py new file mode 100644 index 000000000..6e38fe334 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/synthetic_streaming.py @@ -0,0 +1,403 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +""" +Synthetic streaming dataset for DLRMv3 inference benchmarking. + +This module provides a streaming dataset implementation that loads user interaction +data from pre-generated CSV files with temporal (timestamp) organization, suitable +for simulating real-time recommendation scenarios. +""" + +import csv +import logging +import sys +import time +from typing import Any, Dict, List, Set, Tuple + +import pandas as pd +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import ( + collate_fn, + DLRMv3RandomDataset, + Samples, +) +from generative_recommenders.dlrm_v3.datasets.utils import ( + json_loads, + maybe_truncate_seq, +) +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + +csv.field_size_limit(sys.maxsize) +logger: logging.Logger = logging.getLogger(__name__) + + +class DLRMv3SyntheticStreamingDataset(DLRMv3RandomDataset): + """ + Streaming dataset that loads pre-generated synthetic recommendation data. + + Supports timestamp-based data organization for simulating streaming scenarios + where user interaction histories evolve over time. + + Args: + hstu_config: HSTU model configuration. + ratings_file_prefix: Path prefix for rating data files. + is_inference: Whether dataset is used for inference. + train_ts: Number of timestamps used for training. + total_ts: Total number of timestamps in the data. + num_files: Number of data files (for parallelization). + num_users: Total number of users in the dataset. + num_items: Total number of items in the catalog. + num_categories: Number of item categories. + *args: Additional positional arguments. + **kwargs: Additional keyword arguments. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + ratings_file_prefix: str, + is_inference: bool, + train_ts: int, + total_ts: int, + num_files: int, + num_users: int, + num_items: int, + num_categories: int, + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__(hstu_config=hstu_config, is_inference=is_inference) + self.ratings_file_prefix = ratings_file_prefix + self.file_to_offsets: Dict[int, List[int]] = {} + with open(f"{self.ratings_file_prefix}offset.csv", "r") as file: + reader = csv.reader(file) + for size in range(num_files): + row = next(reader) + assert len(row) == 1 + offset = json_loads(row[0]) + assert len(offset) == num_users // num_files + self.file_to_offsets[size] = offset + self.ts_requests_offsets: List[int] = [] + with open(f"{self.ratings_file_prefix}requests_per_ts_offset.csv", "r") as file: + reader = csv.reader(file) + row = next(reader) + assert len(row) == 1 + self.ts_requests_offsets = json_loads(row[0]) + assert len(self.ts_requests_offsets) == total_ts + self.requests: List[int] = [] + self.ts_to_users_cumsum: Dict[int, List[int]] = {} + with open( + f"{self.ratings_file_prefix}users_cumsum_per_ts.csv", "r" + ) as cumsum_file: + reader = csv.reader(cumsum_file) + ts = 0 + for row in reader: + assert len(row) == 1 + cumsum = json_loads(row[0]) + self.ts_to_users_cumsum[ts] = cumsum + ts += 1 + self.train_ts = train_ts + self.total_ts = total_ts + self.num_files = num_files + self.ts: int = -1 + self.is_inference: bool = False + self.is_eval: bool = False + self.users_per_file: int = num_users // num_files + self.cached_files: Set[str] = set() + self.items_per_category: int = num_items // num_categories + assert hstu_config.action_weights is not None + self.action_weights: List[int] = hstu_config.action_weights + self.items_in_memory: Dict[ + int, Dict[int, Tuple[KeyedJaggedTensor, KeyedJaggedTensor]] + ] = {} + + def get_item_count(self) -> int: + return len(self.requests) + + def load_query_samples(self, sample_list: List[int]) -> None: + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + for idx in sample_list: + data = self.iloc(idx) + sample = self.load_item(data, max_num_candidates) + if self.ts not in self.items_in_memory: + self.items_in_memory[self.ts] = {} + self.items_in_memory[self.ts][idx] = sample + + self.last_loaded = time.time() + + def unload_query_samples(self, sample_list: List[int]) -> None: + self.items_in_memory = {} + + def get_sample(self, id: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + return self.items_in_memory[self.ts][id] + + def get_sample_with_ts( + self, id: int, ts: int + ) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + """ + Get a sample for a specific timestamp. + + Args: + id: Sample identifier. + ts: Timestamp index. + + Returns: + Tuple of (uih_features_kjt, candidates_features_kjt). + """ + return self.items_in_memory[ts][id] + + def get_samples_with_ts(self, id_list: List[int], ts: int) -> Samples: + """ + Get and collate multiple samples for a specific timestamp. + + Args: + id_list: List of sample identifiers. + ts: Timestamp index. + + Returns: + Collated Samples object. + """ + list_samples = [self.get_sample_with_ts(ix, ts) for ix in id_list] + return collate_fn(list_samples) + + def _process_line(self, line: str, user_id: int) -> pd.Series: + """ + Parse a CSV line into a pandas Series with user interaction data. + + Args: + line: CSV line containing user data. + user_id: User identifier. + + Returns: + pd.Series with parsed user interaction history and candidates. + """ + reader = csv.reader([line]) + parsed_line = next(reader) + # total ts + one more eval ts + one base ts so that uih won't be zero + # for each ts, ordered as candidate_ids, candidate_ratings, uih_ids, uih_ratings + assert len(parsed_line) == 4 * (self.total_ts + 2) + uih_item_ids_list = [] + uih_ratings_list = [] + candidate_item_ids = "" + candidate_ratings = "" + if (not self.is_eval) and (not self.is_inference): + assert self.ts < self.train_ts + for i in range(self.ts + 1): + if parsed_line[4 * i]: + uih_item_ids_list.append(parsed_line[2 + 4 * i]) + uih_ratings_list.append(parsed_line[3 + 4 * i]) + candidate_item_ids = parsed_line[4 * (self.ts + 1)] + candidate_ratings = parsed_line[1 + 4 * (self.ts + 1)] + elif self.is_eval: + for i in range(self.ts + 1): + if parsed_line[4 * i]: + uih_item_ids_list.append(parsed_line[2 + 4 * i]) + uih_ratings_list.append(parsed_line[3 + 4 * i]) + candidate_item_ids = parsed_line[4 * (self.ts + 1)] + candidate_ratings = parsed_line[1 + 4 * (self.ts + 1)] + else: + assert self.is_inference is True + assert self.ts >= self.train_ts + for i in range(self.train_ts + 1): + if parsed_line[4 * i]: + uih_item_ids_list.append(parsed_line[2 + 4 * i]) + uih_ratings_list.append(parsed_line[3 + 4 * i]) + for i in range(self.train_ts + 2, self.ts + 2): + if parsed_line[4 * i]: + uih_item_ids_list.append(parsed_line[2 + 4 * i]) + uih_ratings_list.append(parsed_line[3 + 4 * i]) + candidate_item_ids = parsed_line[4 * (self.ts + 2)] + candidate_ratings = parsed_line[1 + 4 * (self.ts + 2)] + uih_item_ids = ",".join(uih_item_ids_list) + uih_ratings = ",".join(uih_ratings_list) + assert candidate_item_ids != "" and candidate_ratings != "" + return pd.Series( + data={ + "user_id": user_id, + "uih_item_ids": uih_item_ids, + "uih_ratings": uih_ratings, + "candidate_item_ids": candidate_item_ids, + "candidate_ratings": candidate_ratings, + } + ) + + def iloc(self, idx: int) -> pd.Series: + """ + Get user data by request index using file offsets for efficient access. + + Args: + idx: Request index within the current timestamp. + + Returns: + pd.Series with parsed user interaction data. + """ + cumsum: List[int] = self.ts_to_users_cumsum[self.ts] + assert cumsum != [] + assert idx < cumsum[-1] + file_idx: int = 0 + while cumsum[file_idx] <= idx: + file_idx += 1 + user_idx = self.requests[idx] + filename = f"{self.ratings_file_prefix}{file_idx}.csv" + with open(filename, "r") as file: + idx = user_idx % self.users_per_file + file.seek(self.file_to_offsets[file_idx][idx]) + line = file.readline() + data = self._process_line(line=line, user_id=user_idx) + return data + + def get_timestamp_uih( + self, data: pd.Series, max_num_candidates: int, size: int + ) -> List[int]: + return [1] * size + + def set_ts(self, ts: int, train_only: bool = False) -> None: + """ + Set the current timestamp and load associated request data. + + Args: + ts: Timestamp index to set. + train_only: Accepted for API parity with the yambda dataset (which + supports a user-level train:eval holdout). This synthetic + dataset has no holdout, so the flag is ignored. + """ + logger.warning(f"Streaming dataset ts set to {ts}") + if ts == self.ts: + return + self.ts = ts + with open( + f"{self.ratings_file_prefix}requests_per_ts.csv", "r" + ) as request_file: + request_file.seek(self.ts_requests_offsets[self.ts]) + line = request_file.readline() + reader = csv.reader([line]) + row = next(reader) + assert len(row) == 1 + requests = json_loads(row[0]) + self.requests = requests + logger.warning(f"DLRMv3SyntheticStreamingDataset: ts={ts} requests loaded") + assert self.ts_to_users_cumsum[self.ts][-1] == len(self.requests) + logger.warning( + f"DLRMv3SyntheticStreamingDataset: ts={ts} users_cumsum={self.ts_to_users_cumsum[self.ts]}" + ) + + def load_item( + self, data: pd.Series, max_num_candidates: int + ) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + """ + Load and process a single user's data into KeyedJaggedTensors. + + Converts parsed user data into feature tensors suitable for model input, + including truncation to maximum sequence lengths. + + Args: + data: pd.Series with user interaction history and candidates. + max_num_candidates: Maximum number of candidates to include. + + Returns: + Tuple of (uih_features_kjt, candidates_features_kjt). + """ + ids_uih = json_loads(data.uih_item_ids) + ids_candidates = json_loads(data.candidate_item_ids) + ratings_uih = json_loads(data.uih_ratings) + ratings_candidates = json_loads(data.candidate_ratings) + timestamps_uih = self.get_timestamp_uih( + data=data, + max_num_candidates=max_num_candidates, + size=len(ids_uih), + ) + assert len(ids_uih) == len(timestamps_uih), ( + "history len differs from timestamp len." + ) + assert len(ids_uih) == len(ratings_uih), ( + f"history len {len(ids_uih)} differs from ratings len {len(ratings_uih)}." + ) + assert len(ids_candidates) == len(ratings_candidates), ( + f"candidates len {len(ids_candidates)} differs from ratings len {len(ratings_candidates)}." + ) + + ids_uih = maybe_truncate_seq(ids_uih, self._max_uih_len) + ratings_uih = maybe_truncate_seq(ratings_uih, self._max_uih_len) + timestamps_uih = maybe_truncate_seq(timestamps_uih, self._max_uih_len) + ids_candidates = maybe_truncate_seq(ids_candidates, max_num_candidates) + num_candidates = len(ids_candidates) + ratings_candidates = maybe_truncate_seq(ratings_candidates, max_num_candidates) + action_weights_uih = [ + self.action_weights[int(rating) - 1] for rating in ratings_uih + ] + action_weights_candidates = [ + int(rating >= 3.5) for rating in ratings_candidates + ] + + uih_kjt_values: List[int] = [] + uih_kjt_lengths: List[int] = [] + for name, length in self._contextual_feature_to_max_length.items(): + uih_kjt_values.append(data[name]) + uih_kjt_lengths.append(length) + + uih_seq_len = len(ids_uih) + dummy_watch_times_uih = [0 for _ in range(uih_seq_len)] + item_category_ids = [id // self.items_per_category for id in ids_uih] + extend_uih_kjt_values: List[int] = ( + ids_uih + + ratings_uih + + timestamps_uih + + action_weights_uih + + dummy_watch_times_uih + + item_category_ids + ) + uih_kjt_values.extend(extend_uih_kjt_values) + uih_kjt_lengths.extend( + [ + uih_seq_len + for _ in range( + len(self._uih_keys) - len(self._contextual_feature_to_max_length) + ) + ] + ) + + dummy_query_time = 0 if timestamps_uih == [] else max(timestamps_uih) + uih_kjt_values.append(dummy_query_time) + uih_kjt_lengths.append(1) + uih_features_kjt: KeyedJaggedTensor = KeyedJaggedTensor( + keys=self._uih_keys + ["dummy_query_time"], + lengths=torch.tensor(uih_kjt_lengths).long(), + values=torch.tensor(uih_kjt_values).long(), + ) + + candidates_kjt_lengths = num_candidates * torch.ones(len(self._candidates_keys)) + item_candidate_category_ids = [ + id // self.items_per_category for id in ids_candidates + ] + candidates_kjt_values = ( + ids_candidates + + ratings_candidates + + [dummy_query_time] * num_candidates # item_query_time + + action_weights_candidates + + [1] * num_candidates # item_dummy_watchtime + + item_candidate_category_ids + ) + candidates_features_kjt: KeyedJaggedTensor = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=candidates_kjt_lengths.detach().clone().long(), + values=torch.tensor(candidates_kjt_values).long(), + ) + return uih_features_kjt, candidates_features_kjt diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/utils.py new file mode 100644 index 000000000..aeca75d41 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/utils.py @@ -0,0 +1,146 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +""" +Utility functions for dataset processing. + +This module provides helper functions for parsing and processing data +in the DLRMv3 dataset pipeline. +""" + +import json +import struct +from typing import Dict, List, Sequence, Tuple + +import numpy as np +import xxhash + + +def json_loads( + x: str | int | List[int], +) -> List[int]: + """ + Parse a JSON-like string into a list of integers. + + Handles multiple input formats including JSON arrays, comma-separated + strings, and single values. + + Args: + x: Input that can be a JSON array string, a single integer, + or already a list of integers. + + Returns: + List of integers parsed from the input. + """ + if isinstance(x, str): + if x[0] != "[" and x[-1] != "]": + x = "[" + x + "]" + y = json.loads(x) + else: + y = x + y_list = [y] if type(y) == int else list(y) + return y_list + + +def separate_uih_candidates( + x: str | int | List[int], + candidates_max_seq_len: int, +) -> Tuple[List[int], List[int]]: + """ + Separate a sequence into user interaction history (UIH) and candidates. + + Splits the input sequence such that the last `candidates_max_seq_len` + elements become candidates and the rest become UIH. + + Args: + x: Input sequence as JSON string, single int, or list of ints. + candidates_max_seq_len: Number of items at the end to use as candidates. + + Returns: + Tuple of (uih, candidates) where both are lists of integers. + """ + if isinstance(x, str): + if x[0] != "[" and x[-1] != "]": + x = "[" + x + "]" + y = json.loads(x) + else: + y = x + y_list = [y] if type(y) == int else list(y) + candidates, uih = ( + y_list[-candidates_max_seq_len:], + y_list[:-candidates_max_seq_len], + ) + return uih, candidates + + +def maybe_truncate_seq( + y: List[int], + max_seq_len: int, +) -> List[int]: + """ + Truncate a sequence if it exceeds the maximum length. + + Args: + y: Input sequence to potentially truncate. + max_seq_len: Maximum allowed sequence length. + + Returns: + The input sequence, truncated to max_seq_len if necessary. + """ + y_len = len(y) + if y_len > max_seq_len: + y = y[:max_seq_len] + return y + + +def xxhash_cross( + anchor: Dict[str, int], + keys: Sequence[str], + table_size: int, + salt: int = 0, +) -> int: + """xxhash64(seed=salt) over little-endian int64 concat(anchor[k] for k in keys), mod table_size. + + Bit-identical to primus_dlrm.data.hashing.cross_hash_nway — embedding rows + are interchangeable with Primus-trained ones. + """ + n = len(keys) + assert n >= 2, f"xxhash_cross needs >=2 keys, got {n}" + digest = xxhash.xxh64(seed=salt) + digest.update(struct.Struct(f"<{n}q").pack(*(int(anchor[k]) for k in keys))) + return digest.intdigest() % table_size + + +def xxhash_cross_batch( + arr_by_key: Dict[str, np.ndarray], + keys: Sequence[str], + table_size: int, + salt: int = 0, +) -> np.ndarray: + """Vectorised xxhash_cross over equal-length int64 arrays (one per key).""" + n = len(keys) + assert n >= 2 + cols = [np.asarray(arr_by_key[k], dtype=np.int64).ravel() for k in keys] + length = cols[0].shape[0] + for c in cols: + assert c.shape[0] == length + pack = struct.Struct(f"<{n}q").pack + digest_cls = xxhash.xxh64 + out = np.empty(length, dtype=np.int64) + for i in range(length): + d = digest_cls(seed=salt) + d.update(pack(*(int(c[i]) for c in cols))) + out[i] = d.intdigest() % table_size + return out diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py new file mode 100644 index 000000000..00b22cff9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/datasets/yambda.py @@ -0,0 +1,1066 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 + +# pyre-unsafe +""" +Yambda dataset for the DLRMv3 HSTU `modules/` path. + +Reads the parquets produced by `dlrm_v3/preprocess_public_data.py +--dataset yambda-`. Each sample is one anchor LISTEN event with: + * label = (played_ratio >= LISTEN_PLUS_THRESHOLD) — the listen_plus bit + * a chronologically interleaved 3-pool history (listen+/like/skip), with + pool identity tagged per-position in `action_weight` (bits 1/2/4) + * 7 pre-hashed cross-feature ids exposed as length-1 contextual entries +""" + +import logging +import mmap as _mmap_mod +import os +import time +from pathlib import Path +from typing import Dict, List, Optional, Sequence, Tuple, Union + +import numpy as np +import polars as pl +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import DLRMv3RandomDataset +from generative_recommenders.dlrm_v3.datasets.utils import xxhash_cross +from generative_recommenders.modules.dlrm_hstu import DlrmHSTUConfig +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + +logger = logging.getLogger(__name__) + + +def _load_npy_readonly(path: Union[str, Path]) -> np.ndarray: + # MAP_SHARED + PROT_READ so the kernel does not charge the mapping against + # vm.overcommit_memory=2 limits. numpy's mmap_mode='r' uses MAP_PRIVATE and + # reserves per-process commit; at 8 ranks × ~190 GB store, that OOMs. + path = Path(path) + with open(path, "rb") as f: + version = np.lib.format.read_magic(f) + if version[0] == 1: + shape, _, dtype = np.lib.format.read_array_header_1_0(f) + else: + shape, _, dtype = np.lib.format.read_array_header_2_0(f) + offset = f.tell() + fd = os.open(str(path), os.O_RDONLY) + try: + buf = _mmap_mod.mmap(fd, 0, access=_mmap_mod.ACCESS_READ) + finally: + os.close(fd) + arr = np.ndarray(shape, dtype=dtype, buffer=buf, offset=offset) + arr.flags.writeable = False + return arr + + +def _uid_unit_hash(uids: np.ndarray, salt: int) -> np.ndarray: + """Deterministic uniform-in-[0,1) hash of user ids (splitmix64 finalizer). + + Pure function of (uid, salt): the same uid always maps to the same value, + so the train/eval user split is identical across processes, ranks, and + crash/resume — the property the no-leakage holdout relies on. Vectorized + uint64 arithmetic wraps mod 2**64 (defined for unsigned), so we silence the + benign overflow warnings. + """ + GOLDEN = np.uint64(0x9E3779B97F4A7C15) + M1 = np.uint64(0xBF58476D1CE4E5B9) + M2 = np.uint64(0x94D049BB133111EB) + s30, s27, s31 = np.uint64(30), np.uint64(27), np.uint64(31) + with np.errstate(over="ignore"): + z = uids.astype(np.uint64) + GOLDEN + np.uint64(salt & 0xFFFFFFFFFFFFFFFF) + z = (z ^ (z >> s30)) * M1 + z = (z ^ (z >> s27)) * M2 + z = z ^ (z >> s31) + # Top 53 bits -> uniform [0, 1) double (same trick numpy uses for randoms). + return (z >> np.uint64(11)).astype(np.float64) * (1.0 / 9007199254740992.0) + +# Yambda event-type encoding written by preprocess_public_data.py. +LISTEN_TYPE = 0 +LIKE_TYPE = 1 +LISTEN_PLUS_THRESHOLD = 50 + +# Action-weight bits (must match hstu_config.action_weights = [1, 2, 4]). +LP_BIT = 1 +LIKE_BIT = 2 +SKIP_BIT = 4 + + +class _FlatEventStore: + """Per-user flat event index built from the preprocessed sessions parquet. + + Reads `train_sessions.parquet` and explodes per-session arrays into flat + numpy columns + per-user `(start, end)` index arrays. Cache-compatible + layout, but writes nothing (rebuilds from parquet each construction). + """ + + # On-disk column layout. + _MMAP_COLS = ( + "flat_uid", "flat_item_ids", "flat_timestamps", + "flat_event_types", "flat_played_ratio", + "flat_is_listen_plus", "flat_is_like", "flat_is_skip", + "flat_is_organic", + "user_start", "user_end", "unique_uids", + ) + + def __init__(self, sessions_df: pl.DataFrame) -> None: + logger.info("Building flat event store from sessions...") + sorted_sessions = sessions_df.sort(["uid", "session_id"]) + exploded = sorted_sessions.explode( + ["item_ids", "timestamps", "event_types", "is_organic", "played_ratio_pct"] + ) + + self.flat_uid: np.ndarray = exploded["uid"].to_numpy().astype(np.int64) + self.flat_item_ids: np.ndarray = exploded["item_ids"].to_numpy().astype(np.int64) + self.flat_timestamps: np.ndarray = exploded["timestamps"].to_numpy().astype(np.int64) + self.flat_event_types: np.ndarray = exploded["event_types"].to_numpy().astype(np.int64) + self.flat_played_ratio: np.ndarray = exploded["played_ratio_pct"].to_numpy().astype(np.float32) + self.flat_is_organic: np.ndarray = exploded["is_organic"].to_numpy().astype(np.int8) + np.nan_to_num(self.flat_played_ratio, copy=False, nan=0.0) + + is_listen = self.flat_event_types == LISTEN_TYPE + self.flat_is_listen_plus: np.ndarray = is_listen & ( + self.flat_played_ratio >= LISTEN_PLUS_THRESHOLD + ) + self.flat_is_like: np.ndarray = self.flat_event_types == LIKE_TYPE + self.flat_is_skip: np.ndarray = is_listen & ( + self.flat_played_ratio < LISTEN_PLUS_THRESHOLD + ) + + uid_changes = np.where(np.diff(self.flat_uid) != 0)[0] + 1 + starts = np.concatenate([[0], uid_changes]) + ends = np.concatenate([uid_changes, [len(self.flat_uid)]]) + uid_vals = self.flat_uid[starts] + max_uid = int(uid_vals.max()) + 1 + self.user_start: np.ndarray = np.full(max_uid, -1, dtype=np.int64) + self.user_end: np.ndarray = np.full(max_uid, -1, dtype=np.int64) + self.user_start[uid_vals] = starts + self.user_end[uid_vals] = ends + self.unique_uids: np.ndarray = uid_vals + self.num_users: int = len(uid_vals) + self.total_events: int = len(self.flat_item_ids) + logger.info( + f"FlatEventStore: {self.total_events:,} events, {self.num_users:,} users" + ) + + @classmethod + def load_mmap(cls, cache_dir: Union[str, Path]) -> "_FlatEventStore": + """Load flat columns by MAP_SHARED+PROT_READ from a prebuilt cache. + All ranks on a node share the same physical pages.""" + import json as _json + cache_dir = Path(cache_dir) + with open(cache_dir / "store_meta.json") as f: + meta = _json.load(f) + store = object.__new__(cls) + for name in cls._MMAP_COLS: + setattr(store, name, _load_npy_readonly(cache_dir / f"{name}.npy")) + store.num_users = int(meta["num_users"]) + store.total_events = int(meta["total_events"]) + logger.info( + f"FlatEventStore mmap from {cache_dir}: " + f"{store.total_events:,} events, {store.num_users:,} users" + ) + return store + + def save_mmap(self, cache_dir: Union[str, Path]) -> None: + """Persist flat columns to disk as .npy, then write a sentinel. + Subsequent runs (any rank, any node sharing the FS) load via mmap.""" + import json as _json + cache_dir = Path(cache_dir) + cache_dir.mkdir(parents=True, exist_ok=True) + for name in self._MMAP_COLS: + np.save(cache_dir / f"{name}.npy", getattr(self, name)) + with open(cache_dir / "store_meta.json", "w") as f: + _json.dump( + {"num_users": self.num_users, "total_events": self.total_events}, f + ) + # Sentinel — readers check this before mmap'ing to avoid partial files. + (cache_dir / "_READY").touch() + logger.info(f"FlatEventStore saved to {cache_dir}") + + +class DLRMv3YambdaDataset(DLRMv3RandomDataset): + """Yambda-5b dataset for the DLRMv3 HSTU modules/ path. + + Args: + hstu_config: DlrmHSTUConfig (must come from `get_hstu_configs("yambda-5b")`). + processed_dir: directory with `train_sessions.parquet` + `item_popularity.npy`. + metadata_dir: directory with `{artist,album}_item_mapping.parquet`. + history_length: UIH cap. Under "interleaved" it is the per-pool cap + (total ≤ 3 * history_length // 3); under "last_n" it is the literal + total number of pooled events kept. + scan_window: how far back to scan when filling each pool. + history_strategy: "interleaved" (equal per-pool L//3 cap, re-interleaved) + or "last_n" (last history_length pooled events, no per-pool split). + cross_specs: list of (name, keys, num_embeddings, salt). Source of truth + in `dlrm_v3/configs.py:YAMBDA_5B_CROSS_SPECS`. + is_inference: passed through to base class. + """ + + def __init__( + self, + hstu_config: DlrmHSTUConfig, + processed_dir: str, + metadata_dir: str, + history_length: int = 2048, + scan_window: int = 20000, + min_history: Optional[int] = None, + history_strategy: str = "interleaved", + cross_specs: Optional[Sequence[Tuple[str, Sequence[str], int, int]]] = None, + cache_dir: Optional[str] = None, + is_inference: bool = False, + streaming_window_seconds: int = 86400, + streaming_sort_within_window: bool = False, + streaming_shuffle_fraction: float = 0.0, + streaming_shuffle_seed: int = 0, + train_split_percentage: float = 1.0, + split_salt: int = 0, + *args, + **kwargs, + ) -> None: + super().__init__(hstu_config=hstu_config, is_inference=is_inference) + self._processed_dir: str = processed_dir + self._metadata_dir: str = metadata_dir + self._history_length: int = history_length + self._scan_window: int = scan_window + # UIH construction strategy: + # "interleaved" (default) — equal history_length//3 cap per behavior + # pool (listen+/like/skip), re-interleaved chronologically. Likes are + # ~1.9% of the corpus so the like pool over-fills relative to its + # natural frequency while the sequence under-fills overall. + # "last_n" — take the last history_length events of ANY pool type with + # no per-pool split. Fills the sequence to ~history_length (higher + # effective length) and lets the like share fall to its natural rate. + # Both exclude dislike/unlike/undislike (no action-weight bit). + if history_strategy not in ("interleaved", "last_n"): + raise ValueError( + f"history_strategy must be 'interleaved' or 'last_n', got " + f"{history_strategy!r}" + ) + self._history_strategy: str = history_strategy + # Minimum prior-event count for a LISTEN event to qualify as an anchor. + # Decoupled from history_length (which is only the gather/truncation cap): + # jagged attention handles short UIH, so we no longer require a full + # history_length of context to include a sample. Default None preserves + # the legacy "need a full history_length of prior events" behavior (which + # dropped ~60% of users); set small (e.g. 1) to include ~all users. + self._min_history: int = ( + history_length if min_history is None else int(min_history) + ) + # Streaming/temporal-order state. Everything here is LAZY: nothing is + # built or read until the first set_ts()/num_windows() call (only the + # streaming-train-eval loop does that), so the default train-eval path + # is byte-for-byte unaffected. + self._streaming_window_seconds: int = streaming_window_seconds + self._streaming_sort_within_window: bool = streaming_sort_within_window + # In-window shuffle dial in [0, 1] to break user-major batching (default + # 0.0 = off, user-major order preserved for page-local mmap scans). Maps to + # a within-segment shuffle with K = round(fraction * per-window train-anchor + # count): 1.0 = full per-element shuffle (max user diversity per batch), + # intermediate = interpolation. Computed from the global anchor count BEFORE + # round-robin striding, so a given fraction yields the same diversity + # regardless of world_size / #nodes / batch_size (config-invariant). The + # permutation is a pure function of (seed, ts) so the per-rank round-robin + # slice + mid-window resume skip stay deterministic across restarts. + self._streaming_shuffle_fraction: float = streaming_shuffle_fraction + self._streaming_shuffle_seed: int = streaming_shuffle_seed + # User-level train:eval split. `train_split_percentage >= 1.0` means no + # holdout (legacy behavior: every anchor is trainable). Otherwise the + # top `1 - train_split_percentage` fraction of users (by a deterministic + # hash of `uid + split_salt`) are held out: NEVER trained, used only to + # build the fixed eval set. The split is a pure function of (uid, salt), + # so it is identical across crash/resume (no leakage on failover). + self._train_split_percentage: float = train_split_percentage + self._split_salt: int = split_salt + # Cache only the (small) fixed eval-holdout index list; the per-window + # train filter is computed on the fly to avoid a full-length mask. + self._eval_holdout_cache: Optional[np.ndarray] = None + self._eval_holdout_cache_key: Optional[Tuple[int, int]] = None + self._active: Optional[np.ndarray] = None + self.is_eval: bool = False + self._anchor_ts: Optional[np.ndarray] = None + self._t_min: Optional[int] = None + self._t_max: Optional[int] = None + self._cache_dir: Optional[str] = cache_dir + self._cross_specs: List[Tuple[str, Tuple[str, ...], int, int]] = [ + (name, tuple(keys), n, s) for (name, keys, n, s) in (cross_specs or []) + ] + assert hstu_config.action_weights is not None + self._action_weights: List[int] = hstu_config.action_weights + + self._load_metadata(metadata_dir) + # Build-once-mmap-many: first rank to arrive acquires the build lock + # and explodes the parquet (one ~190 GB in-memory pass), then writes + # flat .npy columns + _READY sentinel. All ranks (including the + # builder, after dropping its in-memory copy) reload via MAP_SHARED+ + # PROT_READ — kernel shares physical pages across ranks so the steady- + # state per-rank RSS for the dataset is ~0. + if cache_dir is None: + cache_dir = os.path.join(processed_dir, f"hstu_cache_L{history_length}") + self._cache_dir = cache_dir + self._ensure_cache_built(cache_dir, processed_dir, history_length) + self.store: _FlatEventStore = _FlatEventStore.load_mmap(cache_dir) + # Anchor positions depend on min_history (the eligibility floor), not + # just history_length (the gather cap), so they live in a + # min_history-versioned file that shares the flat store. Built + # independently of the _READY sentinel so changing the floor rebuilds + # only this (cheap) array, not the whole 150 GB cache. + self._positions_name: str = self._positions_filename( + history_length, self._min_history + ) + self._ensure_positions_built( + cache_dir, self._positions_name, self._min_history + ) + self._positions: np.ndarray = _load_npy_readonly( + os.path.join(cache_dir, self._positions_name) + ) + logger.info( + f"Yambda dataset ready: {self.store.total_events:,} events, " + f"{len(self._positions):,} training positions" + ) + + @staticmethod + def _positions_filename(history_length: int, min_history: int) -> str: + """Anchor-positions filename. Uses the legacy name when the floor equals + the gather cap (the historical "full history required" behavior) so + existing caches are reused as-is; otherwise a min_history-tagged name.""" + if min_history == history_length: + return f"positions_L{history_length}.npy" + return f"positions_L{history_length}_m{min_history}.npy" + + @staticmethod + def _ensure_positions_built( + cache_dir: str, positions_name: str, min_history: int + ) -> None: + """Build the anchor-positions array for ``min_history`` if absent. + + Anchors are LISTEN events whose user-local offset is >= ``min_history`` + (i.e. the user already has that many prior events). This is decoupled + from the _READY-gated flat-store build so a new floor only rebuilds this + (cheap, ~one int64 scan) array rather than the whole 150 GB cache. + Multi-rank safe via an exclusive lock + atomic rename; all ranks then + mmap the result read-only. + """ + import fcntl + + positions_path = os.path.join(cache_dir, positions_name) + if os.path.exists(positions_path): + return + lock_path = os.path.join(cache_dir, "_positions_lock") + with open(lock_path, "w") as lf: + logger.info(f"Acquiring positions build lock for {positions_path}...") + fcntl.flock(lf, fcntl.LOCK_EX) + try: + if os.path.exists(positions_path): + return + flat_uid = _load_npy_readonly( + os.path.join(cache_dir, "flat_uid.npy") + ) + event_types = _load_npy_readonly( + os.path.join(cache_dir, "flat_event_types.npy") + ) + user_start = _load_npy_readonly( + os.path.join(cache_dir, "user_start.npy") + ) + idx = np.arange(len(flat_uid), dtype=np.int64) + keep = (idx - user_start[flat_uid] >= min_history) & ( + event_types == LISTEN_TYPE + ) + positions = np.where(keep)[0].astype(np.int64) + tmp = positions_path + ".tmp.npy" + np.save(tmp, positions) + os.replace(tmp, positions_path) + logger.info( + f"Wrote {positions_name}: {len(positions):,} anchors " + f"(min_history={min_history})" + ) + finally: + fcntl.flock(lf, fcntl.LOCK_UN) + + @staticmethod + def _ensure_cache_built( + cache_dir: str, processed_dir: str, history_length: int + ) -> None: + """File-locked one-shot build with column-at-a-time explode. + + A naive `pl.read_parquet(...).explode([5 list cols])` peaks at ~1.6 TB + on the 5b dataset (polars holds input list-columns + dense output + + parallel-worker scratch all together). Instead we: + 1) Read parquet + sort once (sorted list-column DF, ~80 GB). + 2) For each output column: select that single list, explode, write + .npy, drop. Bounds incremental peak to one column (~38 GB). + 3) Derive bool flags and indices from the on-disk mmaps. + + Peak RAM: ~150 GB. Steady state across all ranks afterward: ~0 + incremental thanks to MAP_SHARED in load_mmap. + """ + import fcntl + import gc + import json as _json + + ready = os.path.join(cache_dir, "_READY") + if os.path.exists(ready): + return + os.makedirs(cache_dir, exist_ok=True) + lock_path = os.path.join(cache_dir, "_lock") + with open(lock_path, "w") as lf: + logger.info(f"Acquiring build lock for {cache_dir}...") + fcntl.flock(lf, fcntl.LOCK_EX) + try: + if os.path.exists(ready): + return + parquet_path = os.path.join(processed_dir, "train_sessions.parquet") + logger.info( + f"Building flat-event cache from {parquet_path} " + f"(column-at-a-time, ~150 GB peak RAM)" + ) + + # Step 1: read + sort. List columns stay nested at this stage. + sessions = pl.read_parquet(parquet_path).sort(["uid", "session_id"]) + logger.info(f"Sessions sorted: {sessions.shape}") + + # Per-session lengths + uids — used to derive flat_uid via + # np.repeat (cheap) without exploding the whole DF at once. + lengths = ( + sessions.select(pl.col("item_ids").list.len()) + .to_numpy() + .reshape(-1) + .astype(np.int64) + ) + session_uids = sessions["uid"].to_numpy().astype(np.int64) + N = int(lengths.sum()) + num_users = int(np.unique(session_uids).shape[0]) + logger.info(f"Total events: {N:,}, users: {num_users:,}") + + # Step 2: column-at-a-time explode → save → drop. + # uid is per-session scalar; expand via np.repeat. + flat_uid = np.repeat(session_uids, lengths).astype(np.int64) + np.save(os.path.join(cache_dir, "flat_uid.npy"), flat_uid) + del flat_uid, session_uids, lengths + gc.collect() + logger.info("Wrote flat_uid.npy") + + # Derived columns flat_is_listen_plus/like/skip depend on + # event_types + played_ratio. Save those two first, then + # derive the bools from the mmaps. + _list_cols = [ + ("item_ids", "flat_item_ids", np.int64), + ("timestamps", "flat_timestamps", np.int64), + ("event_types", "flat_event_types", np.int64), + ("is_organic", "flat_is_organic", np.int8), + ("played_ratio_pct", "flat_played_ratio", np.float32), + ] + for src_col, dst_name, dtype in _list_cols: + exploded = sessions.select(pl.col(src_col).explode()) + arr = exploded[src_col].to_numpy().astype(dtype, copy=False) + if dtype == np.float32: + np.nan_to_num(arr, copy=False, nan=0.0) + np.save(os.path.join(cache_dir, f"{dst_name}.npy"), arr) + del exploded, arr + gc.collect() + logger.info(f"Wrote {dst_name}.npy") + + # Drop the sessions DF now that all source columns are on disk. + del sessions + gc.collect() + + # Step 3: derive bool flags from the just-written mmaps. + event_types = _load_npy_readonly( + os.path.join(cache_dir, "flat_event_types.npy") + ) + played_ratio = _load_npy_readonly( + os.path.join(cache_dir, "flat_played_ratio.npy") + ) + is_listen = event_types == LISTEN_TYPE + np.save( + os.path.join(cache_dir, "flat_is_listen_plus.npy"), + is_listen & (played_ratio >= LISTEN_PLUS_THRESHOLD), + ) + np.save( + os.path.join(cache_dir, "flat_is_like.npy"), + event_types == LIKE_TYPE, + ) + np.save( + os.path.join(cache_dir, "flat_is_skip.npy"), + is_listen & (played_ratio < LISTEN_PLUS_THRESHOLD), + ) + del is_listen, played_ratio + gc.collect() + logger.info("Wrote flat_is_listen_plus/like/skip.npy") + + # user_start / user_end / unique_uids from flat_uid mmap. + flat_uid = _load_npy_readonly( + os.path.join(cache_dir, "flat_uid.npy") + ) + uid_changes = np.where(np.diff(flat_uid) != 0)[0] + 1 + starts = np.concatenate([[0], uid_changes]) + ends = np.concatenate([uid_changes, [len(flat_uid)]]) + uid_vals = flat_uid[starts] + max_uid = int(uid_vals.max()) + 1 + user_start = np.full(max_uid, -1, dtype=np.int64) + user_end = np.full(max_uid, -1, dtype=np.int64) + user_start[uid_vals] = starts + user_end[uid_vals] = ends + np.save(os.path.join(cache_dir, "user_start.npy"), user_start) + np.save(os.path.join(cache_dir, "user_end.npy"), user_end) + np.save(os.path.join(cache_dir, "unique_uids.npy"), uid_vals) + logger.info("Wrote user_start/end/unique_uids.npy") + + # Positions: LISTEN events with ≥history_length prior history. + # Done now (before dropping user_start) so all sibling ranks + # just mmap the result instead of each running a 75 GB build. + user_start_per_event = user_start[flat_uid] + idx = np.arange(len(flat_uid), dtype=np.int64) + keep = (idx - user_start_per_event >= history_length) & ( + event_types == LISTEN_TYPE + ) + positions = np.where(keep)[0].astype(np.int64) + np.save( + os.path.join(cache_dir, f"positions_L{history_length}.npy"), + positions, + ) + logger.info( + f"Wrote positions_L{history_length}.npy: {len(positions):,}" + ) + del ( + flat_uid, event_types, user_start, user_end, uid_vals, + starts, ends, uid_changes, idx, user_start_per_event, + keep, positions, + ) + gc.collect() + + # Meta + sentinel — written last; readers gate on _READY. + with open(os.path.join(cache_dir, "store_meta.json"), "w") as f: + _json.dump( + {"num_users": num_users, "total_events": N}, f + ) + open(os.path.join(cache_dir, "_READY"), "w").close() + logger.info(f"Cache build complete: {cache_dir}") + finally: + fcntl.flock(lf, fcntl.LOCK_UN) + + def _load_metadata(self, metadata_dir: str) -> None: + item_pop_path = os.path.join(metadata_dir, "item_popularity.npy") + if os.path.exists(item_pop_path): + item_popularity = np.load(item_pop_path) + else: + # Fallback: derive vocab size from the artist+album maps. + item_popularity = None + + artist_map = pl.read_parquet(os.path.join(metadata_dir, "artist_item_mapping.parquet")) + album_map = pl.read_parquet(os.path.join(metadata_dir, "album_item_mapping.parquet")) + n_items = int( + max( + int(artist_map["item_id"].max()) + 1, + int(album_map["item_id"].max()) + 1, + len(item_popularity) if item_popularity is not None else 0, + ) + ) + self.item_to_artist: np.ndarray = np.zeros(n_items, dtype=np.int64) + valid = artist_map.filter(pl.col("item_id") < n_items) + self.item_to_artist[valid["item_id"].to_numpy()] = valid["artist_id"].to_numpy() + self.item_to_album: np.ndarray = np.zeros(n_items, dtype=np.int64) + valid = album_map.filter(pl.col("item_id") < n_items) + self.item_to_album[valid["item_id"].to_numpy()] = valid["album_id"].to_numpy() + self.num_items: int = n_items + + def get_item_count(self) -> int: + # Streaming mode restricts the active set to the current time window; + # otherwise the full (user-major) anchor list is used (train-eval). + if self._active is not None: + return int(len(self._active)) + return int(len(self._positions)) + + def iloc(self, idx: int) -> int: + if self._active is not None: + return int(self._positions[self._active[idx]]) + return int(self._positions[idx]) + + def _ensure_streaming_index(self) -> None: + """Lazily build + mmap the per-anchor target-timestamp array used for + time-windowed streaming. + + Built only on the first ``set_ts()``/``num_windows()`` call, so the + default train-eval path never reads timestamps or writes a new file. + Multi-rank safe via an exclusive file lock + atomic rename; all ranks + then mmap the result read-only (shared physical pages, ~0 anon). + """ + if self._anchor_ts is not None: + return + import fcntl + + assert self._cache_dir is not None + # Target-ts array is per-anchor, so it must track the same min_history + # versioning as the positions file it indexes into. + anchor_path = os.path.join( + self._cache_dir, + self._positions_name.replace("positions_", "anchor_ts_", 1), + ) + if not os.path.exists(anchor_path): + lock_path = os.path.join(self._cache_dir, "_anchor_ts_lock") + with open(lock_path, "w") as lf: + logger.info(f"Acquiring anchor-ts build lock for {anchor_path}...") + fcntl.flock(lf, fcntl.LOCK_EX) + if not os.path.exists(anchor_path): + logger.info( + f"Building {anchor_path}: target ts for " + f"{len(self._positions):,} anchors" + ) + anchor_ts = self.store.flat_timestamps[self._positions] + tmp = anchor_path + ".tmp.npy" + np.save(tmp, anchor_ts) + os.replace(tmp, anchor_path) + del anchor_ts + self._anchor_ts = _load_npy_readonly(anchor_path) + self._t_min = int(self._anchor_ts.min()) + self._t_max = int(self._anchor_ts.max()) + + def num_windows(self) -> int: + """Number of fixed-duration windows spanning [t_min, t_max].""" + self._ensure_streaming_index() + assert self._t_min is not None and self._t_max is not None + span = self._t_max - self._t_min + 1 + w = self._streaming_window_seconds + return int((span + w - 1) // w) + + def window_indices( + self, ts: int, sort_by_time: Optional[bool] = None + ) -> np.ndarray: + """Global anchor indices (into ``_positions``) whose target timestamp is + in window ``ts``: ``[t_min + ts*W, t_min + (ts+1)*W)``. + + Returned in ascending global-index order (user-major), which keeps the + per-sample history scans page-local in the mmap'd event arrays. Used by + the per-window path (via ``set_ts``) and the persistent path (shipped to + workers through the sampler). ``sort_by_time`` defaults to + ``streaming_sort_within_window``. + + Note: an O(log N) variant using a cached argsort of the timestamps was + evaluated but rejected — it doubles resident mmap (sorted-ts + order + permutation, ~52 GB) and that extra residency evicts the event-array + page cache, stalling dataloader workers (NCCL watchdog timeouts). The + O(N) mask here keeps only one ~26 GB array resident and is robust. + """ + self._ensure_streaming_index() + assert self._anchor_ts is not None and self._t_min is not None + w = self._streaming_window_seconds + lo = self._t_min + ts * w + hi = lo + w + idx = np.where((self._anchor_ts >= lo) & (self._anchor_ts < hi))[0] + do_sort = ( + self._streaming_sort_within_window if sort_by_time is None else sort_by_time + ) + if do_sort and idx.size > 0: + idx = idx[np.argsort(self._anchor_ts[idx], kind="stable")] + logger.warning(f"window_indices({ts}): [{lo}, {hi}) -> {idx.size:,} anchors") + return idx.astype(np.int64) + + def _eval_anchor_mask(self, anchor_idx: np.ndarray) -> np.ndarray: + """Bool mask (aligned to ``anchor_idx``) marking held-out eval users. + + Computed on the fly for just this slice of anchors (a window is ~tens of + millions, not the full ~3B ``_positions``), so we never materialize a + full-length mask. ``uid``-hash >= ``train_split_percentage`` -> eval. + """ + uids = self.store.flat_uid[self._positions[anchor_idx]] + return _uid_unit_hash(uids, self._split_salt) >= self._train_split_percentage + + def _shuffle_window(self, idx: np.ndarray, ts: int) -> np.ndarray: + """Optionally break user-major ordering within a train window. + + ``streaming_shuffle_fraction`` (0..1) is the single diversity dial. It + maps to a within-segment shuffle with ``K = round(fraction * N)`` where + ``N`` is this window's train-anchor count: + + - 0.0 -> off: return ``idx`` unchanged (user-major, page-local scans). + - 1.0 -> full per-element shuffle (max user diversity per batch). + - else -> permute WITHIN each contiguous size-K segment (segment order + preserved). A per-rank batch then draws across a bounded user-major + region, so diversity scales with the fraction while the concurrently + touched mmap working set stays within ~one K-segment (page locality). + + Because ``N`` is a property of the dataset/window (not the compute layout) + and the permutation is applied BEFORE the per-rank round-robin striding, a + given fraction yields the same diversity across world_size / #nodes / + batch_size (config-invariant). + + The permutation is a pure function of ``(seed, ts)`` via + ``np.random.default_rng(seed + ts)``, so every (re)run of this window + yields the IDENTICAL order. This keeps the per-rank round-robin slice and + the mid-window resume ``skip_samples`` offset consistent across restarts, + exactly like the unshuffled path. + """ + frac = self._streaming_shuffle_fraction + if idx.size <= 1 or not frac or frac <= 0.0: + return idx + rng = np.random.default_rng(self._streaming_shuffle_seed + ts) + if frac >= 1.0: + return idx[rng.permutation(idx.size)] + # Within-segment shuffle (K = round(fraction * N)): a single vectorized + # lexsort over per-element random keys, stable within each size-K segment + # so elements never cross a segment boundary (bounds the working set). O(N + # log K), run once per window in the background prep thread. + n = idx.size + k = max(1, int(round(frac * n))) + seg = np.arange(n, dtype=np.int64) // k + keys = rng.random(n) + order = np.lexsort((keys, seg)) + return idx[order] + + def train_window_indices(self, ts: int) -> np.ndarray: + """Global anchor indices for TRAIN in window ``ts``: ``window_indices`` + with held-out eval users removed. Identical across resume because + ``window_indices``, the uid hash, and the (seed,ts)-keyed in-window + shuffle are all pure functions, so the per-rank round-robin slice (and + the mid-window skip offset) stay consistent.""" + idx = self.window_indices(ts) + if self._train_split_percentage >= 1.0: + return self._shuffle_window(idx, ts) + kept = idx[~self._eval_anchor_mask(idx)] + logger.warning( + f"train_window_indices({ts}): {idx.size:,} -> {kept.size:,} anchors " + f"(holdout tsp={self._train_split_percentage}, salt={self._split_salt})" + ) + return self._shuffle_window(kept, ts) + + def eval_holdout_indices(self, start_ts: int, num_windows: int = 1) -> np.ndarray: + """Fixed eval set: held-out users' anchors over windows + ``[start_ts, start_ts + num_windows)``. Computed once and cached, so the + SAME anchors are evaluated at every eval step (stable, comparable curve). + With no holdout (tsp>=1.0) this falls back to the full window(s).""" + key = (int(start_ts), int(num_windows)) + if self._eval_holdout_cache is not None and self._eval_holdout_cache_key == key: + return self._eval_holdout_cache + parts: List[np.ndarray] = [] + for ts in range(start_ts, start_ts + max(1, num_windows)): + idx = self.window_indices(ts) + if self._train_split_percentage < 1.0: + idx = idx[self._eval_anchor_mask(idx)] + parts.append(idx) + holdout = ( + np.concatenate(parts).astype(np.int64) + if parts + else np.empty(0, dtype=np.int64) + ) + logger.warning( + f"eval_holdout_indices(start_ts={start_ts}, num_windows={num_windows}): " + f"{holdout.size:,} held-out anchors (tsp={self._train_split_percentage})" + ) + self._eval_holdout_cache = holdout + self._eval_holdout_cache_key = key + return holdout + + def total_train_anchors(self, start_ts: int, num_ts: int) -> int: + """Total TRAIN anchors across windows ``[start_ts, start_ts + num_ts)``. + + A single O(N) pass over the cached ``_anchor_ts`` array (NOT per-window + ``train_window_indices`` scans). Used to convert a "fraction of training + data" eval cadence into a global train-step interval. With a user holdout + (``train_split_percentage`` < 1.0) the held-out eval users are excluded + via the SAME uid hash as ``train_window_indices``, so the count matches + what is actually trained. + + NOTE: this is an UPPER BOUND on the realized train STEP count — the + per-window samplers truncate each window to a multiple of ``world_size`` + and drop the last partial per-rank batch (``drop_last=True``). The small + overcount is acceptable for a cadence knob (it only shifts the eval grid + by a fraction of a window). + """ + self._ensure_streaming_index() + assert self._anchor_ts is not None and self._t_min is not None + if num_ts <= 0: + return 0 + w = self._streaming_window_seconds + lo = self._t_min + start_ts * w + hi = self._t_min + (start_ts + num_ts) * w + in_range = (self._anchor_ts >= lo) & (self._anchor_ts < hi) + if self._train_split_percentage >= 1.0: + total = int(np.count_nonzero(in_range)) + else: + sel = np.where(in_range)[0] + total = int(np.count_nonzero(~self._eval_anchor_mask(sel))) + logger.warning( + f"total_train_anchors(start_ts={start_ts}, num_ts={num_ts}): " + f"{total:,} train anchors (tsp={self._train_split_percentage})" + ) + return total + + def set_ts(self, ts: int, train_only: bool = False) -> None: + """Restrict the active sample set to anchors in window ``ts`` (used by + the per-window-DataLoader path, where ``iloc``/``get_item_count`` index + through ``_active``). + + ``train_only=True`` removes held-out eval users so the non-persistent + TRAIN loader never sees them (closes the leakage path). Forward-only + temporal slicing for streaming train/eval. History for any anchor is + still gathered causally (``scan_start:flat_pos``) and may span earlier + windows, so there is no feature leakage from future events. + """ + self._active = ( + self.train_window_indices(ts) if train_only else self.window_indices(ts) + ) + + def set_active_indices(self, indices: np.ndarray) -> None: + """Restrict the active sample set to an explicit array of global anchor + indices (into ``_positions``). Used by the non-persistent eval path to + iterate the fixed user-holdout set (which spans a window range, not a + single ``ts``).""" + self._active = np.asarray(indices, dtype=np.int64) + + def load_query_samples(self, sample_list) -> None: + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + self.items_in_memory = {} + for idx in sample_list: + flat_pos = self.iloc(idx) + self.items_in_memory[idx] = self._build_sample(flat_pos, max_num_candidates) + self.last_loaded = time.time() + + def get_sample(self, idx: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + if idx in self.items_in_memory: + return self.items_in_memory[idx] + max_num_candidates = ( + self._max_num_candidates_inference + if self._is_inference + else self._max_num_candidates + ) + flat_pos = self.iloc(idx) + return self._build_sample(flat_pos, max_num_candidates) + + @staticmethod + def _empty_history() -> Tuple[ + np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray + ]: + empty = np.empty(0, dtype=np.int64) + return empty, empty, empty, empty, empty + + def _read_scan_window( + self, flat_pos: int, user_start: int + ) -> Optional[ + Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray] + ]: + """Read the causal scan window [scan_start, flat_pos) for an anchor. + Returns (item_ids, timestamps, is_lp, is_like, is_skip) views, or None + if the window is empty.""" + scan_start = max(int(user_start), int(flat_pos) - self._scan_window) + scan_end = int(flat_pos) + if scan_end <= scan_start: + return None + return ( + self.store.flat_item_ids[scan_start:scan_end], + self.store.flat_timestamps[scan_start:scan_end], + self.store.flat_is_listen_plus[scan_start:scan_end], + self.store.flat_is_like[scan_start:scan_end], + self.store.flat_is_skip[scan_start:scan_end], + ) + + def _materialize_history( + self, + keep_local: np.ndarray, + item_ids: np.ndarray, + timestamps: np.ndarray, + is_lp: np.ndarray, + is_like: np.ndarray, + is_skip: np.ndarray, + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Gather item/artist/album/ts + pool-bitmask `weight` for the kept + (chronologically-ordered) local indices.""" + items = item_ids[keep_local] + ts = timestamps[keep_local] + artists = self.item_to_artist[np.clip(items, 0, self.item_to_artist.shape[0] - 1)] + albums = self.item_to_album[np.clip(items, 0, self.item_to_album.shape[0] - 1)] + # Pool bitmask per kept event (LP/LIKE/SKIP are mutually exclusive in + # the source data, but OR is safe and forward-compatible). + weight = np.zeros(keep_local.shape[0], dtype=np.int64) + weight[is_lp[keep_local]] |= LP_BIT + weight[is_like[keep_local]] |= LIKE_BIT + weight[is_skip[keep_local]] |= SKIP_BIT + return items, artists, albums, ts, weight + + def _gather_history( + self, flat_pos: int, user_start: int + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Dispatch UIH construction to the configured strategy.""" + if self._history_strategy == "last_n": + return self._gather_last_n_history(flat_pos, user_start) + return self._gather_interleaved_history(flat_pos, user_start) + + def _gather_interleaved_history( + self, flat_pos: int, user_start: int + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Build a single chronologically-ordered history sequence from the 3 + behavior pools. Each event's `action_weight` carries the pool bitmask + (LP_BIT/LIKE_BIT/SKIP_BIT). Per-pool cap = history_length // 3.""" + L = self._history_length + per_pool = max(1, L // 3) + scan = self._read_scan_window(flat_pos, user_start) + if scan is None: + return self._empty_history() + item_ids, timestamps, is_lp, is_like, is_skip = scan + + # Local indices into the scan window — preserves chronological order + # within each pool and lets us interleave by re-sorting. + idx_all = np.arange(item_ids.shape[0], dtype=np.int64) + lp_idx = idx_all[is_lp][-per_pool:] + like_idx = idx_all[is_like][-per_pool:] + skip_idx = idx_all[is_skip][-per_pool:] + + keep_local = np.concatenate([lp_idx, like_idx, skip_idx]) + if keep_local.size == 0: + return self._empty_history() + + order = np.argsort(keep_local, kind="stable") + keep_local = keep_local[order] + + return self._materialize_history( + keep_local, item_ids, timestamps, is_lp, is_like, is_skip + ) + + def _gather_last_n_history( + self, flat_pos: int, user_start: int + ) -> Tuple[np.ndarray, np.ndarray, np.ndarray, np.ndarray, np.ndarray]: + """Build the UIH from the last `history_length` events of ANY pool type + (listen+/like/skip) with no per-pool split. Vs the interleaved strategy + this fills the sequence to ~history_length (higher effective length) and + lets the like share fall to its natural corpus rate (~1.9%). Events + outside the 3 pools (dislike/unlike/undislike) are excluded as before.""" + L = self._history_length + scan = self._read_scan_window(flat_pos, user_start) + if scan is None: + return self._empty_history() + item_ids, timestamps, is_lp, is_like, is_skip = scan + + member = is_lp | is_like | is_skip + # Last L pooled events, in chronological order (already position-sorted + # within the scan window, so no re-sort is needed). + keep_local = np.arange(item_ids.shape[0], dtype=np.int64)[member][-L:] + if keep_local.size == 0: + return self._empty_history() + + return self._materialize_history( + keep_local, item_ids, timestamps, is_lp, is_like, is_skip + ) + + def _build_sample( + self, flat_pos: int, max_num_candidates: int + ) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + uid = int(self.store.flat_uid[flat_pos]) + user_start = int(self.store.user_start[uid]) + + items, artists, albums, ts, weight = self._gather_history( + flat_pos, user_start + ) + + target_item = int(self.store.flat_item_ids[flat_pos]) + target_artist = int( + self.item_to_artist[target_item] + if target_item < self.item_to_artist.shape[0] + else 0 + ) + target_album = int( + self.item_to_album[target_item] + if target_item < self.item_to_album.shape[0] + else 0 + ) + target_ts = int(self.store.flat_timestamps[flat_pos]) + + played_ratio = float(self.store.flat_played_ratio[flat_pos]) + is_lp = ( + int(self.store.flat_event_types[flat_pos]) == LISTEN_TYPE + and played_ratio >= LISTEN_PLUS_THRESHOLD + ) + # Label encoded into the candidate's action_weight via the LP bit, so + # _get_supervision_labels_and_weights sees the right supervision. + candidate_action_weight = LP_BIT if is_lp else 0 + + cross_id_anchor: Dict[str, int] = { + "uid": uid, + "item_id": target_item, + "artist_id": target_artist, + "album_id": target_album, + "hour_of_day": int((target_ts // 3600) % 24), + "is_organic": int(self.store.flat_is_organic[flat_pos]), + } + cross_ids: Dict[str, int] = { + name: xxhash_cross(cross_id_anchor, list(keys), n, salt) + for (name, keys, n, salt) in self._cross_specs + } + + # ---- Truncate UIH to fit max_seq_len budget ---- + uih_seq_len_budget = ( + self._max_seq_len + - max_num_candidates + - len(self._contextual_feature_to_max_length or {}) + ) + if items.shape[0] > uih_seq_len_budget: + items = items[-uih_seq_len_budget:] + artists = artists[-uih_seq_len_budget:] + albums = albums[-uih_seq_len_budget:] + ts = ts[-uih_seq_len_budget:] + weight = weight[-uih_seq_len_budget:] + uih_seq_len = int(items.shape[0]) + dummy_watch_time = np.zeros(uih_seq_len, dtype=np.int64) + + # ---- Build UIH KJT ---- + # Contextual features (length-1 each) iterated in the same order as + # `_contextual_feature_to_max_length` (matches movielens reference). + uih_kjt_values: List[int] = [] + uih_kjt_lengths: List[int] = [] + for name, length in (self._contextual_feature_to_max_length or {}).items(): + assert length == 1, f"yambda contextuals are length-1, got {name}={length}" + if name == "uid": + uih_kjt_values.append(uid) + else: + uih_kjt_values.append(int(cross_ids[name])) + uih_kjt_lengths.append(1) + + # Sequential features — order must match the trailing entries of + # hstu_uih_feature_names in configs.py: + # item_id, artist_id, album_id, action_weight, action_timestamp, dummy_watch_time + uih_kjt_values.extend(items.tolist()) + uih_kjt_values.extend(artists.tolist()) + uih_kjt_values.extend(albums.tolist()) + uih_kjt_values.extend(weight.tolist()) + uih_kjt_values.extend(ts.tolist()) + uih_kjt_values.extend(dummy_watch_time.tolist()) + n_sequential = len(self._uih_keys) - len(self._contextual_feature_to_max_length or {}) + uih_kjt_lengths.extend([uih_seq_len] * n_sequential) + + dummy_query_time = int(ts[-1]) if uih_seq_len > 0 else target_ts + uih_kjt_values.append(dummy_query_time) + uih_kjt_lengths.append(1) + + uih_features_kjt = KeyedJaggedTensor( + keys=self._uih_keys + ["dummy_query_time"], + lengths=torch.tensor(uih_kjt_lengths, dtype=torch.long), + values=torch.tensor(uih_kjt_values, dtype=torch.long), + ) + + # ---- Build candidates KJT ---- + # Order must match configs.py:hstu_candidate_feature_names exactly: + # item_candidate_id, item_candidate_artist_id, item_candidate_album_id, + # item_query_time, item_action_weight, item_dummy_watchtime + candidates_kjt_lengths = max_num_candidates * torch.ones( + len(self._candidates_keys), dtype=torch.long + ) + candidates_kjt_values: List[int] = ( + [target_item] * max_num_candidates + + [target_artist] * max_num_candidates + + [target_album] * max_num_candidates + + [dummy_query_time] * max_num_candidates + + [candidate_action_weight] * max_num_candidates + + [1] * max_num_candidates # item_dummy_watchtime + ) + candidates_features_kjt = KeyedJaggedTensor( + keys=self._candidates_keys, + lengths=candidates_kjt_lengths, + values=torch.tensor(candidates_kjt_values, dtype=torch.long), + ) + return uih_features_kjt, candidates_features_kjt diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py b/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py new file mode 100644 index 000000000..06b43aa48 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/preprocess_public_data.py @@ -0,0 +1,539 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +import argparse +import json +import logging +import os +import tarfile +from pathlib import Path +from typing import Dict, List, Optional, Tuple +from urllib.request import urlretrieve + +import numpy as np +import pandas as pd + + +logging.basicConfig(level=logging.INFO) +log = logging.getLogger("main") + +""" +Usage: + mkdir -p data/ && python3 preprocess_public_data.py --dataset kuairand-1k + python3 preprocess_public_data.py --dataset yambda-5b --data-path + python3 preprocess_public_data.py --dataset yambda-500m --data-path + python3 preprocess_public_data.py --dataset yambda-50m --data-path +""" + +SUPPORTED_DATASETS = [ + "kuairand-1k", + "kuairand-27k", + "yambda-50m", + "yambda-500m", + "yambda-5b", +] + + +def get_feature_merge_weights(dataset: str = "debug") -> Dict[str, int]: + if "kuairand" in dataset: + return { + "is_click": 1, + "is_like": 2, + "is_follow": 4, + "is_comment": 8, + "is_forward": 16, + "is_hate": 32, + "long_view": 64, + "is_profile_enter": 128, + } + else: + return {"dummy": 1} + + +class DataProcessor: + def __init__( + self, + download_url: str, + data_path: str, + file_name: str, + prefix: str, + ) -> None: + self._download_url = download_url + self._data_path = data_path + self._file_name = file_name + self._prefix = prefix + + def download(self) -> None: + return + + def preprocess(self) -> None: + return + + def file_exists(self, name: str) -> bool: + return os.path.isfile("%s/%s" % (os.getcwd(), name)) + + +class DLRMKuaiRandProcessor(DataProcessor): + def __init__( + self, + download_url: str, + data_path: str, + file_name: str, + prefix: str, + ) -> None: + super().__init__(download_url, data_path, file_name, prefix) + if prefix == "KuaiRand-1K": + self._log_files: List[str] = [ + f"{data_path}{prefix}/data/log_standard_4_08_to_4_21_1k.csv", + f"{data_path}{prefix}/data/log_standard_4_22_to_5_08_1k.csv", + ] + self._user_features_file: str = ( + f"{data_path}{prefix}/data/user_features_1k.csv" + ) + elif prefix == "KuaiRand-27K": + self._log_files: List[str] = [ + f"{data_path}{prefix}/data/log_standard_4_08_to_4_21_27k_part1.csv", + f"{data_path}{prefix}/data/log_standard_4_08_to_4_21_27k_part2.csv", + f"{data_path}{prefix}/data/log_standard_4_22_to_5_08_27k_part1.csv", + f"{data_path}{prefix}/data/log_standard_4_22_to_5_08_27k_part2.csv", + ] + self._user_features_file: str = ( + f"{data_path}{prefix}/data/user_features_27k.csv" + ) + self._output_file: str = f"{data_path}{prefix}/data/processed_seqs.csv" + self._event_merge_weight: Dict[str, int] = get_feature_merge_weights( + prefix.lower() + ) + + def download(self) -> None: + file_path = f"{self._data_path}{self._file_name}" + if not self.file_exists(file_path): + log.info(f"Downloading {self._download_url}") + urlretrieve(self._download_url, file_path) + log.info(f"Downloaded to {file_path}") + with tarfile.open(file_path, "r:*") as tar_ref: + tar_ref.extractall(path=self._data_path) + log.info("Data files extracted") + os.remove(file_path) + log.info("Tar file removed") + + def preprocess(self) -> None: + self.download() + log.info("Preprocessing data...") + seq_cols = [ + "video_id", + "time_ms", + "action_weights", + "play_time_ms", + "duration_ms", + ] + df = None + for idx, log_file in enumerate(self._log_files): + log.info(f"Processing {log_file}...") + log_df = pd.read_csv( + log_file, + delimiter=",", + ) + df_grouped_by_user = log_df.groupby("user_id").agg(list).reset_index() + + for event, weight in self._event_merge_weight.items(): + df_grouped_by_user[event] = df_grouped_by_user[event].apply( + lambda seq: np.where(np.array(seq) == 0, 0, weight) + ) + + events = list(self._event_merge_weight.keys()) + df_grouped_by_user["action_weights"] = df_grouped_by_user.apply( + lambda row: [int(sum(x)) for x in zip(*[row[col] for col in events])], + axis=1, + ) + df_grouped_by_user = df_grouped_by_user[["user_id"] + seq_cols] + + if idx == 0: + df = df_grouped_by_user + else: + df = df.merge(df_grouped_by_user, on="user_id", suffixes=("_x", "_y")) + for col in seq_cols: + df[col] = df.apply( + lambda row: row[col + "_x"] + row[col + "_y"], axis=1 + ) + df = df.drop(columns=[col + "_x", col + "_y"]) + + max_seq_len = df["video_id"].apply(len).max() + min_seq_len = df["video_id"].apply(len).min() + average_seq_len = df["video_id"].apply(len).mean() + log.info(f"{max_seq_len=}, {min_seq_len=}, {average_seq_len=}") + + log.info("Merging user features...") + user_features_df = pd.read_csv(self._user_features_file, delimiter=",") + + def _one_hot_encode(row): + mapping = {category: i + 1 for i, category in enumerate(row.unique())} + row = row.map(mapping) + return row + + for col in [ + "user_active_degree", + "follow_user_num_range", + "fans_user_num_range", + "friend_user_num_range", + "register_days_range", + ]: + user_features_df[col] = _one_hot_encode(user_features_df[col]) + + final_df = pd.merge(df, user_features_df, on="user_id") + final_df.to_csv(self._output_file, index=False, sep=",") + log.info(f"Processed file saved to {self._output_file}") + + +# ---------------------------------------------------------------------------- +# Yambda processor +# ---------------------------------------------------------------------------- +# +# Yambda is hosted on HuggingFace at `yandex/yambda` and comes in three sizes: +# 50m, 500m, 5b. Each size shares the same catalog metadata (embeddings, +# artist/album mappings); only the interaction stream differs. +# +# This processor: +# 1) Downloads `multi_event.parquet` for the chosen size + the catalog +# metadata files via the `datasets` library. +# 2) Encodes event_type strings into uint8. +# 3) Splits temporally into train + test (Global Temporal Split, GTS). +# 4) Builds per-user sessions by inactivity gap. +# 5) Computes item popularity counts. +# 6) Writes the layout expected by `DLRMv3YambdaDataset`: +# +# /processed_/ +# train_sessions.parquet +# test_events.parquet +# session_index.parquet +# item_popularity.npy +# split_meta.json +# +# /shared_metadata/ +# artist_item_mapping.parquet +# album_item_mapping.parquet +# embeddings.parquet (optional; not used by HSTU training) +# +# The HSTU training path then auto-builds an `hstu_cache_L/` mmap under +# `processed_/` on first use. +# ---------------------------------------------------------------------------- + +YAMBDA_HF_REPO = "yandex/yambda" +YAMBDA_SIZES = {"yambda-50m": "50m", "yambda-500m": "500m", "yambda-5b": "5b"} +YAMBDA_METADATA_FILES = ( + "artist_item_mapping", + "album_item_mapping", + "embeddings", +) + +# Yambda timestamps are seconds (rounded to 5s boundaries). +SECONDS_PER_DAY = 86400 +# Polars chunk size for streaming the 5b parquet (~150 GB on disk). +YAMBDA_CHUNK_SIZE = 10_000_000 +EVENT_TYPE_MAP = {"listen": 0, "like": 1, "dislike": 2, "unlike": 3, "undislike": 4} + + +class DLRMYambdaProcessor(DataProcessor): + """Download + preprocess Yambda (50m / 500m / 5b) for DLRMv3YambdaDataset.""" + + def __init__( + self, + data_path: str, + size: str, + session_gap_seconds: int = 1800, + train_days: int = 300, + gap_minutes: int = 30, + test_days: int = 1, + ) -> None: + assert size in {"50m", "500m", "5b"}, f"unknown yambda size {size}" + super().__init__( + download_url="", # download is via HuggingFace `datasets` lib + data_path=data_path.rstrip("/") + "/", + file_name=f"{size}/multi_event.parquet", + prefix=f"yambda-{size}", + ) + self._size: str = size + self._raw_dir: Path = Path(self._data_path) / "raw" + self._processed_dir: Path = Path(self._data_path) / f"processed_{size}" + self._shared_dir: Path = Path(self._data_path) / "shared_metadata" + self._session_gap_seconds: int = session_gap_seconds + self._train_days: int = train_days + self._gap_minutes: int = gap_minutes + self._test_days: int = test_days + + def download(self) -> None: + try: + from datasets import DatasetDict, load_dataset + except ImportError as e: + raise ImportError( + "Downloading Yambda requires the `datasets` package " + "(`pip install datasets`)." + ) from e + + self._raw_dir.mkdir(parents=True, exist_ok=True) + self._shared_dir.mkdir(parents=True, exist_ok=True) + + # Size-specific interaction stream. + event_path = self._raw_dir / self._size / "multi_event.parquet" + if not event_path.exists(): + event_path.parent.mkdir(parents=True, exist_ok=True) + log.info( + f"Downloading multi_event.parquet for {self._size} " + f"from {YAMBDA_HF_REPO} ..." + ) + ds = load_dataset( + YAMBDA_HF_REPO, + data_dir=f"flat/{self._size}", + data_files="multi_event.parquet", + ) + assert isinstance(ds, DatasetDict) + ds["train"].to_parquet(str(event_path)) + log.info(f"Saved {event_path}") + else: + log.info(f"Already exists: {event_path}") + + # Catalog metadata files (shared across sizes). + for name in YAMBDA_METADATA_FILES: + shared_path = self._shared_dir / f"{name}.parquet" + if shared_path.exists(): + log.info(f"Already exists: {shared_path}") + continue + log.info(f"Downloading {name}.parquet from {YAMBDA_HF_REPO} ...") + ds = load_dataset(YAMBDA_HF_REPO, data_files=f"{name}.parquet") + assert isinstance(ds, DatasetDict) + ds["train"].to_parquet(str(shared_path)) + log.info(f"Saved {shared_path}") + + def preprocess(self) -> None: + self.download() + try: + import polars as pl + except ImportError as e: + raise ImportError( + "Yambda preprocessing requires polars " + "(`pip install polars-u64-idx` is recommended for the 5b " + "variant — stock polars overflows its 32-bit row index)." + ) from e + + self._processed_dir.mkdir(parents=True, exist_ok=True) + event_path = self._raw_dir / self._size / "multi_event.parquet" + + log.info(f"Loading multi_event from {event_path} ...") + events = self._load_events(pl, event_path) + log.info(f"Loaded {len(events):,} events") + + events = self._encode_event_types(pl, events) + t_min = int(events["timestamp"].min()) + t_max = int(events["timestamp"].max()) + log.info( + f"Timestamp range: {t_min}..{t_max} " + f"({(t_max - t_min) / SECONDS_PER_DAY:.1f} days)" + ) + + train_start, train_end, test_start, test_end = self._split_boundaries(t_max) + log.info( + f"GTS train=[{train_start},{train_end}) gap=[{train_end},{test_start}) " + f"test=[{test_start},{test_end})" + ) + train_events, test_events = self._temporal_split( + pl, events, train_start, train_end, test_start, test_end + ) + log.info( + f"Train: {len(train_events):,} events, Test: {len(test_events):,} events" + ) + + gap_units = self._session_gap_seconds # 1 unit = 1 second + sessions = self._build_sessions(pl, train_events, gap_units) + log.info(f"Built {len(sessions):,} sessions") + + session_index = self._build_session_index(pl, sessions) + log.info(f"Session index covers {len(session_index):,} users") + + item_popularity = self._compute_item_popularity(train_events) + + sessions.write_parquet(str(self._processed_dir / "train_sessions.parquet")) + test_events.write_parquet(str(self._processed_dir / "test_events.parquet")) + session_index.write_parquet(str(self._processed_dir / "session_index.parquet")) + np.save(self._processed_dir / "item_popularity.npy", item_popularity) + + with open(self._processed_dir / "split_meta.json", "w") as f: + json.dump( + { + "size": self._size, + "t_min": t_min, + "t_max": t_max, + "train_start": train_start, + "train_end": train_end, + "test_start": test_start, + "test_end": test_end, + "train_days": self._train_days, + "gap_minutes": self._gap_minutes, + "test_days": self._test_days, + "session_gap_seconds": self._session_gap_seconds, + "num_train_events": int(len(train_events)), + "num_test_events": int(len(test_events)), + "num_sessions": int(len(sessions)), + "num_users": int(len(session_index)), + }, + f, + indent=2, + ) + log.info(f"Preprocessing complete: {self._processed_dir}") + + # ------- helpers -------- + + def _load_events(self, pl, parquet_path: Path): + # 5b is too large to load in one polars pass on most boxes (~150 GB + # peak in-RAM with eager read). Stream in 10M-row chunks for safety. + if self._size == "5b": + log.info(f"Streaming load (chunk_size={YAMBDA_CHUNK_SIZE:,})...") + lf = pl.scan_parquet(parquet_path) + n = lf.select(pl.len()).collect().item() + log.info(f"Total rows: {n:,}") + chunks = [] + for off in range(0, n, YAMBDA_CHUNK_SIZE): + chunk = lf.slice(off, YAMBDA_CHUNK_SIZE).collect() + chunks.append(chunk) + log.info(f" loaded {off:,}..{off + len(chunk):,}") + return pl.concat(chunks) + return pl.read_parquet(parquet_path) + + def _encode_event_types(self, pl, events): + dt = events["event_type"].dtype + if dt == pl.Utf8 or isinstance(dt, (pl.Categorical, pl.Enum)): + events = events.with_columns( + pl.col("event_type") + .cast(pl.Utf8) + .replace_strict(EVENT_TYPE_MAP) + .cast(pl.UInt8) + .alias("event_type") + ) + return events + + def _split_boundaries(self, t_max: int) -> Tuple[int, int, int, int]: + test_end = t_max + test_start = test_end - self._test_days * SECONDS_PER_DAY + train_end = test_start - self._gap_minutes * 60 + train_start = train_end - self._train_days * SECONDS_PER_DAY + return train_start, train_end, test_start, test_end + + def _temporal_split(self, pl, events, train_start, train_end, test_start, test_end): + train = events.filter( + (pl.col("timestamp") >= train_start) & (pl.col("timestamp") < train_end) + ) + test_all = events.filter( + (pl.col("timestamp") >= test_start) & (pl.col("timestamp") < test_end) + ) + # Test users must also appear in train (next-item prediction setup). + train_users = train.select("uid").unique() + test = test_all.join(train_users, on="uid", how="inner") + return train, test + + def _build_sessions(self, pl, events, session_gap_units: int): + sorted_events = events.sort(["uid", "timestamp"]) + return ( + sorted_events + .with_columns( + ( + (pl.col("timestamp").diff().fill_null(0) > session_gap_units) + .cast(pl.UInt32) + .cum_sum() + ) + .over("uid") + .alias("session_id") + ) + .group_by(["uid", "session_id"]) + .agg( + pl.col("item_id").alias("item_ids"), + pl.col("timestamp").alias("timestamps"), + pl.col("event_type").alias("event_types"), + pl.col("is_organic").alias("is_organic"), + pl.col("played_ratio_pct").alias("played_ratio_pct"), + pl.col("track_length_seconds").alias("track_length_seconds"), + ) + .sort(["uid", "session_id"]) + ) + + def _build_session_index(self, pl, sessions): + return ( + sessions + .with_columns(pl.col("item_ids").list.len().alias("session_len")) + .group_by("uid") + .agg( + pl.col("session_id").alias("session_ids"), + pl.col("session_len").alias("session_lens"), + pl.col("session_len").cum_sum().alias("session_offsets"), + ) + .sort("uid") + ) + + def _compute_item_popularity(self, train_events) -> np.ndarray: + counts = ( + train_events + .group_by("item_id") + .len() + .sort("item_id") + ) + max_item = int(counts["item_id"].max()) + popularity = np.zeros(max_item + 1, dtype=np.int64) + popularity[counts["item_id"].to_numpy()] = counts["len"].to_numpy() + return popularity + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset", + choices=SUPPORTED_DATASETS, + required=True, + help="dataset", + ) + parser.add_argument( + "--data-path", + default="data/", + help=( + "Root directory for raw + processed data. KuaiRand defaults to " + "the existing `data/` convention; Yambda defaults to `data/` too " + "but is commonly overridden to a shared filesystem location with " + "enough space for the 5b variant (~500 GB)." + ), + ) + args = parser.parse_args() + + data_path = args.data_path.rstrip("/") + "/" + + if args.dataset == "kuairand-1k": + DLRMKuaiRandProcessor( + download_url="https://zenodo.org/records/10439422/files/KuaiRand-1K.tar.gz", + data_path=data_path, + file_name="KuaiRand-1K.tar.gz", + prefix="KuaiRand-1K", + ).preprocess() + elif args.dataset == "kuairand-27k": + DLRMKuaiRandProcessor( + download_url="https://zenodo.org/records/10439422/files/KuaiRand-27K.tar.gz", + data_path=data_path, + file_name="KuaiRand-27K.tar.gz", + prefix="KuaiRand-27K", + ).preprocess() + elif args.dataset in YAMBDA_SIZES: + DLRMYambdaProcessor( + data_path=data_path, + size=YAMBDA_SIZES[args.dataset], + ).preprocess() + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/streaming_synthetic_data.py b/recommendation_v4/generative_recommenders/dlrm_v3/streaming_synthetic_data.py new file mode 100644 index 000000000..bb9e508af --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/streaming_synthetic_data.py @@ -0,0 +1,664 @@ +# pyre-strict +""" +Streaming synthetic data generator for DLRMv3. + +This module generates synthetic streaming recommendation data for benchmarking +and testing purposes. It creates user-item interaction histories with timestamps, +ratings, and category-based item distributions. +""" + +import csv +import logging +import math +import multiprocessing +import os +import random +import shutil +import time +from typing import Dict, List, Tuple + +import numpy as np + +logger: logging.Logger = logging.getLogger(__name__) + + +class StreamingSyntheticDataGenerator: + """ + Generator for streaming synthetic recommendation data. + + Creates realistic user-item interaction data with temporal dynamics, + category preferences, and rating distributions for benchmarking + recommendation systems. + + Args: + num_categories: Number of item categories. + categories_per_user: Number of categories each user is interested in. + num_users: Total number of users to generate. + num_items: Total number of items in the catalog. + num_timestamps: Number of time periods in the streaming data. + avg_samples_per_item: Average number of interactions per item. + train_ratio: Fraction of timestamps used for training. + user_sampling_ratio: Probability of sampling a user at each timestamp. + num_eval_candidates: Number of candidates for evaluation. + num_inference_candidates: Number of candidates for inference. + debug: If True, use deterministic ratings for debugging. + rank: Process rank for distributed generation. + """ + + def __init__( + self, + num_categories: int, + categories_per_user: int, + num_users: int, + num_items: int, + num_timestamps: int, + avg_samples_per_item: int, + train_ratio: float, + user_sampling_ratio: float, + num_eval_candidates: int, + num_inference_candidates: int, + debug: bool = False, + rank: int = 0, + ) -> None: + self.num_categories = num_categories + self.categories_per_user = categories_per_user + self.num_users = num_users + self.num_items = num_items + self.num_timestamps = num_timestamps + self.avg_samples_per_item = avg_samples_per_item + self.avg_seq_len_per_timestamp = int( + num_items * avg_samples_per_item / num_users / num_timestamps + ) + self.items_per_category: int = num_items // num_categories + self.category_to_start_end_item_idx: Dict[int, Tuple[int, int]] = {} + for i in range(num_categories): + start_idx = i * self.items_per_category + end_idx = (i + 1) * self.items_per_category + self.category_to_start_end_item_idx[i] = (start_idx, end_idx) + self.alpha_range = (1, 500) + self.min_seq_len: int = num_eval_candidates + 1 + self.train_ratio = train_ratio + self.num_eval_candidates = num_eval_candidates + self.num_inference_candidates = num_inference_candidates + self.debug = debug + self.total_cnt = 0 + self.rank = rank + logger.warning(f"rank {self.rank}: start generating item rating") + np.random.seed(1001) + self.item_rating = np.random.choice( # pyre-ignore [4] + [5.0, 4.0, 3.0, 2.0, 1.0], size=num_items, p=[0.2, 0.25, 0.25, 0.2, 0.1] + ) + logger.warning(f"rank {self.rank}: finish generating item rating") + self.user_sampling_ratio = user_sampling_ratio + + def generate_one_timestamp( + self, + category_to_cnt: Dict[int, int], + categories: List[int], + t: int, + id: int, + output_folder: str, + uih_seq_len: int, + eval: bool, + inference: bool, + file_idx: int, + ts_buffers: Dict[int, List[int]], + ) -> Tuple[List[int], List[float], List[int], List[float], Dict[int, int]]: + """ + Generate interaction data for a single user at one timestamp. + + Args: + category_to_cnt: Running count of interactions per category. + categories: Categories this user is interested in. + t: Current timestamp index. + id: User ID. + output_folder: Output directory for files. + uih_seq_len: Length of user interaction history to generate. + eval: Whether this is for evaluation. + inference: Whether this is for inference. + file_idx: File index for output. + ts_buffers: Buffer for timestamp data. + + Returns: + Tuple of (uih_item_ids, uih_ratings, candidate_ids, candidate_ratings, + updated_category_counts). + """ + if t >= 0 and (not eval): + if t not in ts_buffers: + ts_buffers[t] = [] + ts_buffers[t].append(id) + seq_len: int = self.num_inference_candidates if inference else uih_seq_len + self.total_cnt += seq_len + alpha = random.randint(self.alpha_range[0], self.alpha_range[1]) + total_cnt = sum(category_to_cnt.values()) + p = np.array( + [ + (alpha / len(categories) + category_to_cnt[c]) / (alpha + total_cnt) + for c in categories + ] + ) + item_categories = np.random.choice(categories, size=seq_len, p=p) + unique, counts = np.unique(item_categories, return_counts=True) + for cat, cnt in zip(unique, counts): + category_to_cnt[cat] += int(cnt) + sample_end_idx = int( + self.items_per_category * max((t + 1), 1) / self.num_timestamps + ) + sample_inds = np.random.randint(0, sample_end_idx, size=seq_len) + offsets = np.array( + [self.category_to_start_end_item_idx[cat][0] for cat in item_categories] + ) + sample_inds = sample_inds + offsets + num_categories = len(categories) + quarter = num_categories // 4 + half = num_categories // 2 + three_quarter = num_categories // 4 * 3 + category_to_ratings = {} + cos1 = math.cos(t * math.pi / 4) + cos2 = math.cos((t + 2) * math.pi / 4) + cos3 = math.cos((t + 4) * math.pi / 4) + for i, cat in enumerate(categories): + if i < quarter: + if self.debug: + ratings = np.full(seq_len, 5.0) + else: + ratings = np.random.choice( + [4.5 + 0.5 * cos1, 4.0 + 0.5 * cos2], + size=seq_len, + p=[0.8, 0.2], + ) + elif i < half: + if self.debug: + ratings = np.full(seq_len, 4.0) + else: + ratings = np.random.choice( + [4.5 + 0.5 * cos1, 4.0 + 0.5 * cos2, 3.5 + 0.5 * cos3], + size=seq_len, + p=[0.1, 0.8, 0.1], + ) + elif i < three_quarter: + if self.debug: + ratings = np.full(seq_len, 3.0) + else: + ratings = np.random.choice( + [3.5 + 0.5 * cos1, 3.0 + 0.5 * cos2, 2.5 + 0.5 * cos3], + size=seq_len, + p=[0.1, 0.8, 0.1], + ) + else: + if self.debug: + ratings = np.full(seq_len, 2.0) + else: + ratings = np.random.choice( + [2.5 + 0.5 * cos1, 2.0 + 0.5 * cos2, 1.5 + 0.5 * cos3], + size=seq_len, + p=[0.1, 0.8, 0.1], + ) + category_to_ratings[cat] = ratings + sample_inds = sample_inds.tolist() + sample_ratings = [ + ( + category_to_ratings[item_categories[i]][i] + + self.item_rating[sample_inds[i]] + ) + / 2 + for i in range(seq_len) + ] + if not inference: + sub_indices = random.sample(range(seq_len), self.num_eval_candidates) + sample_candidate_inds = [sample_inds[i] for i in sub_indices] + sample_candidate_ratings = [sample_ratings[i] for i in sub_indices] + sample_uih_inds = sample_inds + sample_uih_ratings = sample_ratings + else: + sub_indices = random.sample(range(seq_len), uih_seq_len) + sample_uih_inds = [sample_inds[i] for i in sub_indices] + sample_uih_ratings = [sample_ratings[i] for i in sub_indices] + sample_candidate_inds = sample_inds + sample_candidate_ratings = sample_ratings + return ( + sample_uih_inds, + sample_uih_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) + + def gen_rand_seq_len(self) -> int: + """ + Generate a random sequence length from a Gaussian distribution. + + Returns: + Sequence length, guaranteed to be at least min_seq_len. + """ + seq_len = round( + random.gauss( + self.avg_seq_len_per_timestamp, self.avg_seq_len_per_timestamp // 4 + ) + ) + seq_len = self.min_seq_len if seq_len < self.min_seq_len else seq_len + return seq_len + + def get_timestamp_sample(self, t: int) -> int: + """ + Determine if a user should be sampled at this timestamp. + + Args: + t: Timestamp index. Base timestamp (-1) is always sampled. + + Returns: + 1 if the user should be sampled, 0 otherwise. + """ + if t == -1: + sample = 1 + else: + sample = np.random.choice( + [1, 0], + size=1, + p=[self.user_sampling_ratio, 1 - self.user_sampling_ratio], + )[0] + return sample + + def generate_one_user( + self, + id: int, + output_folder: str, + file_idx: int, + ts_buffers: Dict[int, List[int]], + ) -> List[str]: + """ + Generate complete interaction history for one user. + + Creates training, evaluation, and inference data for a single user + across all timestamps. + + Args: + id: User ID. + output_folder: Output directory. + file_idx: File index for output. + ts_buffers: Buffer for timestamp metadata. + + Returns: + List of CSV row values for this user's data. + """ + categories = random.sample(range(self.num_categories), self.categories_per_user) + category_to_cnt = {c: 0 for c in categories} + out_list: List[str] = [] + # t = -1 as base UIH + ( + sample_inds, + sample_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) = self.generate_one_timestamp( + category_to_cnt=category_to_cnt, + categories=categories, + t=-1, + id=id, + output_folder=output_folder, + uih_seq_len=self.gen_rand_seq_len(), + eval=False, + inference=False, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + out_list.append(",".join([str(ind) for ind in sample_candidate_inds])) + out_list.append(",".join([str(rat) for rat in sample_candidate_ratings])) + out_list.append(",".join([str(ind) for ind in sample_inds])) + out_list.append(",".join([str(rat) for rat in sample_ratings])) + # train + for t in range(int(self.num_timestamps * self.train_ratio)): + if self.get_timestamp_sample(t): + ( + sample_inds, + sample_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) = self.generate_one_timestamp( + category_to_cnt=category_to_cnt, + categories=categories, + t=t, + id=id, + output_folder=output_folder, + uih_seq_len=self.gen_rand_seq_len(), + eval=False, + inference=False, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + out_list.append(",".join([str(ind) for ind in sample_candidate_inds])) + out_list.append( + ",".join([str(rat) for rat in sample_candidate_ratings]) + ) + out_list.append(",".join([str(ind) for ind in sample_inds])) + out_list.append(",".join([str(rat) for rat in sample_ratings])) + else: + out_list += ["", "", "", ""] + # eval + ( + sample_inds, + sample_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) = self.generate_one_timestamp( + category_to_cnt=category_to_cnt, + categories=categories, + t=int(self.num_timestamps * self.train_ratio), + id=id, + output_folder=output_folder, + uih_seq_len=self.num_eval_candidates, + eval=True, + inference=False, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + out_list.append(",".join([str(ind) for ind in sample_candidate_inds])) + out_list.append(",".join([str(rat) for rat in sample_candidate_ratings])) + out_list.append(",".join([str(ind) for ind in sample_inds])) + out_list.append(",".join([str(rat) for rat in sample_ratings])) + # inference + for t in range( + int(self.num_timestamps * self.train_ratio), self.num_timestamps + ): + if self.get_timestamp_sample(t): + ( + sample_inds, + sample_ratings, + sample_candidate_inds, + sample_candidate_ratings, + category_to_cnt, + ) = self.generate_one_timestamp( + category_to_cnt=category_to_cnt, + categories=categories, + t=t, + id=id, + output_folder=output_folder, + uih_seq_len=self.gen_rand_seq_len(), + eval=False, + inference=True, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + out_list.append(",".join([str(ind) for ind in sample_candidate_inds])) + out_list.append( + ",".join([str(rat) for rat in sample_candidate_ratings]) + ) + out_list.append(",".join([str(ind) for ind in sample_inds])) + out_list.append(",".join([str(rat) for rat in sample_ratings])) + else: + out_list += ["", "", "", ""] + return out_list + + def write_dataset( + self, output_folder: str, num_files: int, file_idx: int, seed: int + ) -> None: + """ + Write dataset for a single file partition. + + Args: + output_folder: Output directory path. + num_files: Total number of files in the dataset. + file_idx: Index of this file partition. + seed: Random seed for reproducibility. + """ + t0 = time.time() + num_users_per_file = self.num_users // num_files + user_id: int = num_users_per_file * file_idx + random.seed(seed + file_idx) + np.random.seed(seed + file_idx) + # Buffer timestamp data in memory to avoid excessive file I/O + ts_buffers: Dict[int, List[int]] = {} + output_file = output_folder + f"{file_idx}.csv" + with open(output_file, "w") as file: + writer = csv.writer(file) + for i in range(num_users_per_file): + out_list = self.generate_one_user( + id=user_id, + output_folder=output_folder, + file_idx=file_idx, + ts_buffers=ts_buffers, + ) + user_id += 1 + writer.writerow(out_list) + if i % 10000 == 0: + logger.warning( + f"rank {self.rank}: Done with users {i} for file {file_idx + 1} / {num_files}, total_cnt = {self.total_cnt}, spends {time.time() - t0} seconds." + ) + # Write buffered timestamp data after all users are processed + for ts, user_ids in ts_buffers.items(): + ts_file = output_folder + f"ts_{file_idx}_{ts}.csv" + with open(ts_file, "w") as f: + writer = csv.writer(f) + for uid in user_ids: + writer.writerow([uid]) + logger.warning( + f"rank {self.rank}: Wrote {len(ts_buffers)} timestamp files for file {file_idx}" + ) + + +def worker( + rank: int, + world_size: int, + num_files: int, + num_users: int, + num_items: int, + num_categories: int, + categories_per_user: int, + num_timestamps: int, + avg_samples_per_item: int, + num_eval_candidates: int, + num_inference_candidates: int, + train_ratio: float, + user_sampling_ratio: float, + output_folder: str, +) -> None: + """ + Worker function for parallel data generation. + + Each worker generates a subset of the dataset files. + + Args: + rank: Worker rank. + world_size: Total number of workers. + num_files: Total files to generate. + num_users: Total users in dataset. + num_items: Total items in catalog. + num_categories: Number of item categories. + categories_per_user: Categories per user. + num_timestamps: Number of time periods. + avg_samples_per_item: Average interactions per item. + num_eval_candidates: Eval candidates count. + num_inference_candidates: Inference candidates count. + train_ratio: Training data fraction. + user_sampling_ratio: User sampling probability. + output_folder: Output directory. + """ + generator = StreamingSyntheticDataGenerator( + num_categories=num_categories, + categories_per_user=categories_per_user, + num_users=num_users, + num_items=num_items, + num_timestamps=num_timestamps, + avg_samples_per_item=avg_samples_per_item, + train_ratio=train_ratio, + user_sampling_ratio=user_sampling_ratio, + num_eval_candidates=num_eval_candidates, + num_inference_candidates=num_inference_candidates, + debug=False, + rank=rank, + ) + num_files_per_rank = num_files // world_size + file_indices = [i + rank * num_files_per_rank for i in range(num_files_per_rank)] + for file_idx in file_indices: + logger.warning(f"rank {rank}: start generating file {file_idx}") + generator.write_dataset( + output_folder=output_folder, + num_files=num_files, + file_idx=file_idx, + seed=1001, + ) + logger.warning(f"rank {rank}: finish generating file {file_idx}") + + +def write_offset(output_folder: str, num_files: int, num_users: int) -> None: + """ + Write file byte offsets for random access to user data. + + Creates an offset.csv file containing byte positions for each user + within their respective data files. + + Args: + output_folder: Directory containing data files. + num_files: Number of data files. + num_users: Total number of users. + """ + with open(output_folder + "offset.csv", "a") as output_file: + writer = csv.writer(output_file) + for i in range(num_files): + input_file = output_folder + f"{i}.csv" + offsets = [] + with open(input_file, "r") as f: + while True: + offset = f.tell() + line = f.readline() + if not line: + break + offsets.append(offset) + assert len(offsets) == num_users // num_files, ( + f"num_users {num_users // num_files} != {len(offsets)}" + ) + logger.warning(f"offsets for file {i} finished") + writer.writerow([",".join([str(offset) for offset in offsets])]) + + +def write_ts_metadata(output_folder: str, total_ts: int, num_files: int) -> None: + """ + Write timestamp metadata for streaming simulation. + + Creates files tracking which users are active at each timestamp + and cumulative counts for efficient streaming access. + + Args: + output_folder: Output directory path. + total_ts: Total number of timestamps. + num_files: Number of data files. + """ + with open(output_folder + "requests_per_ts.csv", "w") as file_requests: + with open(output_folder + "users_cumsum_per_ts.csv", "w") as file_cumsum: + requests_writer = csv.writer(file_requests) + cumsum_writer = csv.writer(file_cumsum) + for ts in range(total_ts): + requests = [] + num_users_per_file = [] + for file in range(num_files): + with open(f"{output_folder}ts_{file}_{ts}.csv", "r") as file: + reader = csv.reader(file) + size = 0 + for row in reader: + requests.append(int(row[0])) + size += 1 + num_users_per_file.append(size) + cumsum = np.cumsum(num_users_per_file).tolist() + assert cumsum[-1] == len(requests) + requests_writer.writerow([",".join([str(r) for r in requests])]) + cumsum_writer.writerow([",".join([str(s) for s in cumsum])]) + logger.warning(f"ts {ts} finished") + with open( + output_folder + "requests_per_ts_offset.csv", "w" + ) as file_requests_offset: + writer = csv.writer(file_requests_offset) + input_file = output_folder + "requests_per_ts.csv" + offsets = [] + with open(input_file, "r") as f: + while True: + offset = f.tell() + line = f.readline() + if not line: + break + offsets.append(offset) + assert len(offsets) == total_ts, f"total_ts {total_ts} != {len(offsets)}" + logger.warning("offsets for file requests_per_ts.csv finished") + writer.writerow([",".join([str(offset) for offset in offsets])]) + + +def copy_sub_dataset(src_folder: str) -> None: + """ + Copy a subset of dataset files for quick testing. + + Creates a sampled_data subdirectory with essential files. + + Args: + src_folder: Source folder containing full dataset. + """ + dst_folder = src_folder + "sampled_data/" + files_to_copy = [ + "0.csv", + "offset.csv", + "requests_per_ts.csv", + "requests_per_ts_offset.csv", + "users_cumsum_per_ts.csv", + ] + os.makedirs(dst_folder, exist_ok=True) + for filename in files_to_copy: + src_path = os.path.join(src_folder, filename) + dst_path = os.path.join(dst_folder, filename) + shutil.copy2(src_path, dst_path) + logger.warning("Files copied successfully.") + + +def main() -> None: + """ + Main entry point for synthetic data generation. + + Configures and launches parallel workers to generate a complete + streaming recommendation dataset. + """ + processes = [] + num_files = 100 + num_users = 5_000_000 + num_items = 1_000_000_000 + num_categories = 128 + categories_per_user = 4 + num_timestamps = 100 + avg_samples_per_item = 50 + num_eval_candidates = 32 + num_inference_candidates = 2048 + train_ratio = 0.9 + user_sampling_ratio = 0.7 + world_size = 5 + username = os.getlogin() + output_folder = f"/home/{username}/data/streaming-100b/" + for i in range(world_size): + p = multiprocessing.Process( + target=worker, + args=( + i, + world_size, + num_files, + num_users, + num_items, + num_categories, + categories_per_user, + num_timestamps, + avg_samples_per_item, + num_eval_candidates, + num_inference_candidates, + train_ratio, + user_sampling_ratio, + output_folder, + ), + ) + processes.append(p) + p.start() + for p in processes: + p.join() + write_offset(output_folder, num_files, num_users) + write_ts_metadata(output_folder, num_timestamps, num_files) + copy_sub_dataset(src_folder=output_folder) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/tests/test_lifetime_auc_resume.py b/recommendation_v4/generative_recommenders/dlrm_v3/tests/test_lifetime_auc_resume.py new file mode 100644 index 000000000..3d5bbd1ee --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/tests/test_lifetime_auc_resume.py @@ -0,0 +1,146 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict + +""" +Round-trip correctness test for ``LifetimeAUCMetricComputation`` checkpoint +serialization. + +Background: torchrec's ``AUCMetricComputation`` registers its +PREDICTIONS/LABELS/WEIGHTS buffers with ``persistent=False``, so the default +``state_dict()`` returns them empty and a separate ``_num_samples`` counter is +dropped too. Without the overrides on ``LifetimeAUCMetricComputation`` every +checkpoint resume would silently restart the lifetime AUC from an empty buffer. + +These tests assert: + 1. update -> compute == A; state_dict -> load_state_dict on a fresh metric -> + compute == A (buffers survive the round trip). + 2. ``_num_samples`` round-trips exactly (required so the next update() does + not take the init-sentinel branch and desync windowed eviction). + 3. The shared-blob path (buffers stripped) leaves a fresh metric empty, so the + per-rank artifact is the sole authority for the trailing buffer. + +Runs in <1s on CPU. Skipped automatically if torchrec is unavailable. +""" + +import unittest + +import torch + +try: + from generative_recommenders.dlrm_v3.utils import LifetimeAUCMetricComputation + + _HAVE_DEPS = True +except Exception: # pragma: no cover - import guard for envs without torchrec + _HAVE_DEPS = False + + +def _make_metric(n_tasks: int = 1, window: int = 10_000_000): + return LifetimeAUCMetricComputation( + my_rank=0, + batch_size=128, + n_tasks=n_tasks, + window_size=window, + ) + + +def _feed(metric, preds, labels, weights) -> None: + metric.update( + predictions=preds, + labels=labels, + weights=weights, + ) + + +@unittest.skipUnless(_HAVE_DEPS, "torchrec / generative_recommenders not importable") +class LifetimeAUCResumeTest(unittest.TestCase): + def setUp(self) -> None: + torch.manual_seed(0) + self.n_tasks = 1 + self.n = 4096 + self.preds = torch.rand(self.n_tasks, self.n) + self.labels = (torch.rand(self.n_tasks, self.n) > 0.5).float() + self.weights = torch.ones(self.n_tasks, self.n) + + def _compute_value(self, metric) -> float: + reports = metric._compute() + return float(reports[0].value.flatten()[0].item()) + + def test_state_dict_round_trip_preserves_auc(self) -> None: + m = _make_metric(self.n_tasks) + _feed(m, self.preds, self.labels, self.weights) + auc_a = self._compute_value(m) + n_a = m.lifetime_sample_count() + self.assertEqual(n_a, self.n) + + sd = m.state_dict() + + fresh = _make_metric(self.n_tasks) + fresh.load_state_dict(sd) + auc_b = self._compute_value(fresh) + + self.assertEqual(fresh.lifetime_sample_count(), self.n) + self.assertAlmostEqual(auc_a, auc_b, places=6) + + def test_num_samples_round_trips(self) -> None: + m = _make_metric(self.n_tasks) + _feed(m, self.preds, self.labels, self.weights) + sd = m.state_dict() + fresh = _make_metric(self.n_tasks) + fresh.load_state_dict(sd) + self.assertEqual(fresh._num_samples, m._num_samples) + + def test_continued_update_after_resume_matches_uninterrupted(self) -> None: + # Splitting a stream and resuming in the middle must equal feeding it all + # at once (this is what fails when _num_samples is not restored). + half = self.n // 2 + p1, p2 = self.preds[:, :half], self.preds[:, half:] + l1, l2 = self.labels[:, :half], self.labels[:, half:] + w1, w2 = self.weights[:, :half], self.weights[:, half:] + + ref = _make_metric(self.n_tasks) + _feed(ref, p1, l1, w1) + _feed(ref, p2, l2, w2) + auc_ref = self._compute_value(ref) + + part = _make_metric(self.n_tasks) + _feed(part, p1, l1, w1) + resumed = _make_metric(self.n_tasks) + resumed.load_state_dict(part.state_dict()) + _feed(resumed, p2, l2, w2) + auc_resumed = self._compute_value(resumed) + + self.assertAlmostEqual(auc_ref, auc_resumed, places=6) + + def test_blob_state_dict_strips_buffers(self) -> None: + from generative_recommenders.dlrm_v3.checkpoint import ( + _metric_blob_state_dict, + ) + + m = _make_metric(self.n_tasks) + _feed(m, self.preds, self.labels, self.weights) + blob = _metric_blob_state_dict(m) + prefix = LifetimeAUCMetricComputation._LIFETIME_KEY_PREFIX + self.assertFalse(any(k.startswith(prefix) for k in blob.keys())) + + # A fresh metric loaded from the stripped blob must NOT have history — + # the per-rank artifact is the only source of the trailing buffer. + fresh = _make_metric(self.n_tasks) + fresh.load_state_dict(blob) + self.assertEqual(fresh.lifetime_sample_count(), 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py new file mode 100644 index 000000000..5470d4e39 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/_env_bootstrap.py @@ -0,0 +1,38 @@ +"""Gin-driven env-var bootstrap. + +Some env vars must be set *before* certain modules import (e.g. Triton's +`@triton.autotune` decorator reads `TRITON_FULL_AUTOTUNE` at module load +time, well before `gin.parse_config_file` runs in the default ordering). + +`apply_env_bootstrap()` is `@gin.configurable`, so the gin file becomes the +canonical source of truth. `train_ranker.py` parses gin with +`skip_unknown=True` early in `_main_func`, calls this function to push the +bindings into `os.environ`, then does the heavy imports. +""" + +import logging +import os +from typing import Optional + +import gin + +logger: logging.Logger = logging.getLogger(__name__) + + +@gin.configurable +def apply_env_bootstrap( + TRITON_FULL_AUTOTUNE: Optional[bool] = None, +) -> None: + # A pre-set environment variable wins over the gin binding. The pinned + # triton configs are MI350X-specific, so a different GPU arch (e.g. B200 + # sm_100) sets TRITON_FULL_AUTOTUNE=1 in the launcher environment to + # re-enable the full autotune search WITHOUT editing this (AMD-default) + # gin file. Cross-cluster launchers thus stay config-as-code via env. + if "TRITON_FULL_AUTOTUNE" in os.environ: + logger.info( + "env bootstrap: honoring pre-set TRITON_FULL_AUTOTUNE=%s (overrides gin binding)", + os.environ["TRITON_FULL_AUTOTUNE"], + ) + elif TRITON_FULL_AUTOTUNE is not None: + os.environ["TRITON_FULL_AUTOTUNE"] = "1" if TRITON_FULL_AUTOTUNE else "0" + logger.info("env bootstrap: TRITON_FULL_AUTOTUNE=%s", os.environ["TRITON_FULL_AUTOTUNE"]) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/debug.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/debug.gin new file mode 100644 index 000000000..9261dc222 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/debug.gin @@ -0,0 +1,35 @@ +batch_size = 16 +dataset = "debug" + +# model parameters +make_model.dataset = %dataset + +# dense model optimizer +dense_optimizer_factory_and_class.learning_rate = 0.001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +sparse_optimizer_factory_and_class.learning_rate = 0.001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = 0.75 + +# train loop variables +train_loop.num_batches = 10 +train_loop.num_epochs = 1000 +train_loop.output_trace = True +train_loop.metric_log_frequency = 10 + +# logger variables +MetricsLogger.tensorboard_log_path = "/tmp/tensorboard_log_path.log" diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin new file mode 100644 index 000000000..188d6cad6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/gin/yambda_5b.gin @@ -0,0 +1,875 @@ +# Per-rank batch size. Env-overridable so a denser per-sample shape can fit in +# HBM by lowering it without editing gin. Default 1024 preserves prior runs; +# streaming train+eval both read this macro (make_*streaming_dataloader.batch_size). +batch_size = @bs/env_int() +bs/env_int.key = "BATCH_SIZE" +bs/env_int.default = 1024 +# Dataloader parallelism. Env-overridable so a perf sweep can probe whether the +# shuffle steady-state cost is CPU-gather latency (hidden by more workers) vs +# GPU-side embedding work (not). Defaults preserve prior behavior (4 / 8). +num_workers = @nw/env_int() +nw/env_int.key = "NUM_WORKERS" +nw/env_int.default = 4 +prefetch_factor = @pf/env_int() +pf/env_int.key = "PREFETCH_FACTOR" +pf/env_int.default = 8 +dataset = "yambda-5b" + +# model parameters +make_model.dataset = %dataset +make_model.bf16_training = True +# HSTU attention/compute backend: "TRITON" (fused, flash-style — low HBM) or +# "PYTORCH" (unfused; materializes the dense [B,H,N,N] score tensor, ~32 GiB at +# N=2048/bs=1024). TRITON validated on MI350/ROCm. The HSTU_HAMMER_KERNEL env +# var, if set, overrides this binding for one-off runs. +make_model.hammer_kernel = "TRITON" + +# False = use pinned triton kernel configs (deterministic; whether that's +# the fast or slow equilibrium depends on which config was pinned for the +# current training shape + GPU). For a NEW training config (new shape, +# new GPU, new triton/torch version), set True and run with +# TRITON_PRINT_AUTOTUNING=1 to discover the fast configs, then update the +# pinned constants in ops/triton/_autotune_pinning.py call sites. +apply_env_bootstrap.TRITON_FULL_AUTOTUNE = False + +# ============================================================================= +# $SEED — global RNG seed for reproducible MODEL INITIALIZATION. +# +# WHAT IT CONTROLS (all weight init is a deterministic function of $SEED): +# 1. Dense parameters (HSTU transformer blocks, MLPs, action embeddings, the +# postprocessor). seed_everything() seeds python/numpy/torch/torch.cuda +# with $SEED right before make_model(). The SAME seed is set on every rank, +# so dense weights are initialized identically across ranks AND reproducibly +# run-to-run. +# 2. Sparse embedding tables (item_id, artist_id, album_id, uid, the cross +# tables). These are materialized on-device while DMP shards the model, NOT +# in make_model(), so seed_everything() alone does not pin them. Two +# mechanisms tie them to $SEED: +# (a) make_optimizer_and_shard() RE-SEEDS torch/torch.cuda from $SEED +# immediately before DistributedModelParallel(...), so the fused +# FBGEMM TBE init (which draws from the global RNG on-device) is +# reproducible. This is the path that actually applies here. +# (b) get_embedding_table_config() also attaches a per-table seeded +# init_fn (seed derived from sha256($SEED, table_name)) for the +# eager/non-meta code path. It no-ops on the meta device that DMP +# uses, so (a) is the effective guarantee for this setup. +# 3. Any other seeded RNG consumers (e.g. dropout's init-time draws). +# +# SCOPE ("Tier 1"): reproducible for a FIXED sharding plan — same GPU/world +# size AND same planner output. It is NOT invariant to changing the GPU count +# or sharding (the per-shard draw boundaries move). The init DISTRIBUTION is +# unchanged from stock (uniform +/-1/sqrt(num_embeddings), or a table's +# explicit weight_init_min/max), so $SEED affects determinism, not quality. +# +# VERIFY: INIT_CHECKSUM=1 (OFF by default) logs a per-table fingerprint + a +# one-line "[init-checksum] SEED=.. digest=.." right after DMP. Two builds with +# the same $SEED + plan print the same digest; different seeds differ. It is OFF +# by default because the fp64 per-shard reductions materialize a full fp64 copy +# of each local embedding shard (>150 GiB for the big tables), leaving almost no +# HBM headroom after sharding and OOMing the build on any node with residual +# memory. Enable only for an explicit reproducibility check (ideally a clean +# node / smaller batch). +# +# WHAT IT DOES *NOT* CONTROL (separate, independent knobs): +# - Streaming data order / shuffle permutation -> $STREAMING_SHUFFLE_SEED +# (get_dataset.streaming_shuffle_seed, below). +# - Train/eval holdout user split -> $SPLIT_SALT (below). +# So holding $STREAMING_SHUFFLE_SEED + $SPLIT_SALT fixed and varying ONLY $SEED +# isolates the effect of model initialization (a clean init-seed A/B / sweep). +# +# PARSE NOTE: seed_everything() runs right before make_model() in train_ranker +# (after the full gin parse), so this binding resolves in the second parse where +# env_int is registered. Override per-run via $SEED. +# Default 1 gives a fixed, reproducible seed each run; override $SEED to vary it +# (set $SEED = -1 to draw a fresh random seed per run). +seed_everything.seed = @seed/env_int() +seed/env_int.key = "SEED" +seed/env_int.default = 1 + +# $DECORRELATE_DROPOUT — re-seed torch/cuda with $SEED + rank after init so HSTU +# dropout masks differ per data-parallel rank. 1 = on, 0 = identical masks (default). +decorrelate_runtime_rng.enabled = @drr/env_int() +drr/env_int.key = "DECORRELATE_DROPOUT" +drr/env_int.default = 0 + +# dense model optimizer +# Learning rate is env-overridable via $DENSE_LR (default 1e-6). +dense_optimizer_factory_and_class.learning_rate = @dlr/env_float() +dlr/env_float.key = "DENSE_LR" +dlr/env_float.default = 0.000001 +dense_optimizer_factory_and_class.optimizer_name = "Adam" +dense_optimizer_factory_and_class.momentum = 0 +dense_optimizer_factory_and_class.weight_decay = 0 +dense_optimizer_factory_and_class.eps = 1e-8 +dense_optimizer_factory_and_class.betas = (0.95, 0.999) + +# sparse model optimizer +# Learning rate is env-overridable via $SPARSE_LR (default 1e-6). +sparse_optimizer_factory_and_class.learning_rate = @slr/env_float() +slr/env_float.key = "SPARSE_LR" +slr/env_float.default = 0.000001 +sparse_optimizer_factory_and_class.optimizer_name = "RowWiseAdagrad" +sparse_optimizer_factory_and_class.momentum = 0 +sparse_optimizer_factory_and_class.weight_decay = 0 +sparse_optimizer_factory_and_class.eps = 1e-8 +sparse_optimizer_factory_and_class.betas = (0.95, 0.999) + +# Gradient clipping for the STREAMING path (clips dense params; sparse tables +# use a fused optimizer and are unaffected). Env-overridable via $GRAD_CLIP_NORM. +# Default 1.0 = ON (max_norm=1.0); set 0.0 via $GRAD_CLIP_NORM to disable. +streaming_train_eval_loop.grad_clip_norm = @gcn/env_float() +gcn/env_float.key = "GRAD_CLIP_NORM" +gcn/env_float.default = 1.0 + +# ============================================================================ +# LR WARMUP (streaming train/eval path) +# ============================================================================ +# WHAT IT DOES +# Linearly ramps the learning rate from 0 up to the configured base LR over +# the first N GLOBAL train steps, then holds flat at the base LR for the rest +# of the run: +# +# effective_lr(gstep) = base_lr * min(1.0, gstep / N) +# +# The ramp scales BOTH learning-rate channels in lockstep: +# 1. the dense / non-fused optimizer (the `optimizer` passed to the loop), and +# 2. the in-backward FUSED sparse-embedding optimizer (model.fused_optimizer), +# i.e. the per-table embedding LR that DOMINATES convergence for this +# model. The warmed LR is also pushed into the FBGEMM TBE backward kernel +# (via set_learning_rate), so the embedding backward actually sees the +# ramped value in the SAME step (not one step late). +# +# WHY IT EXISTS +# Warmup tames the early-step trajectory divergence that a large LR causes on +# freshly-initialized weights/embeddings. That early divergence is a major +# source of run-to-run variance (CV). Warmup lets us use a higher base LR (or +# stretch the gentle early-rise region) without paying the early-instability +# penalty. It does NOT reduce plateau-region CV (that is set by data ordering). +# +# RESUME SAFETY +# The ramp is keyed off the monotonic, checkpoint-restored global_step, so it +# is identical on a cold start and on every relaunch: +# - a run that crashes mid-warmup resumes at the correct multiplier; +# - a run already past step N is a pure no-op (LRs already hold base). +# +# HOW TO ENABLE / DISABLE +# Controlled by the $LR_WARMUP_STEPS environment variable (read at startup): +# - DISABLED (default): unset $LR_WARMUP_STEPS, or set it to 0. +# => warmup code early-returns; base LRs are NEVER mutated; behavior is +# byte-identical to the pre-warmup baseline. This is the default so +# existing/baseline runs are unaffected. +# - ENABLED: export LR_WARMUP_STEPS= (N = number of global steps to ramp) +# e.g. export LR_WARMUP_STEPS=1500 # short warmup (~0.5% of data @ GB=8192) +# export LR_WARMUP_STEPS=16000 # long warmup (~5.7% of data @ GB=8192) +# Rule of thumb: data fraction spent ramping = N * global_batch / total_train. +# +# NOTE: this only changes the LR SCHEDULE. The base LR itself is set separately +# via DENSE_LR / SPARSE_LR; warmup never overrides those base values. +streaming_train_eval_loop.lr_warmup_steps = @lrw/env_int() +lrw/env_int.key = "LR_WARMUP_STEPS" +lrw/env_int.default = 24000 # 24k-step LR warmup (CV-reduction default) + +# ---------------------------------------------------------------------------- +# LR WARMUP FLOOR / STARTING LR (only meaningful when $LR_WARMUP_STEPS > 0) +# ---------------------------------------------------------------------------- +# WHAT IT DOES +# Sets the LR the ramp STARTS from at global step 0 instead of starting from +# zero, turning the schedule into a "partial warmup" that ramps from `start` +# up to the base LR over $LR_WARMUP_STEPS, then holds flat: +# +# effective_lr(gstep) = start + (base_lr - start) * min(1.0, gstep / N) +# +# `start` is applied PER param group and clamped to <= that group's base LR +# (a floor >= base means "no ramp" for that group -> it starts at base). Like +# the main warmup, the floor scales both the dense optimizer and the fused +# sparse-embedding optimizer, and is pushed into the FBGEMM TBE kernel. +# +# WHY IT EXISTS +# With a long warmup the first several thousand steps run at a near-zero LR +# (e.g. at gstep 1k of a 24k ramp, LR ~= base/24 ~= 4e-9) and barely train. +# A small nonzero floor reclaims that near-dead head of the ramp, banking +# extra effective learning (integral rises from 1/2*N*base to +# 1/2*N*(base+start)) to reach a target AUC in fewer steps. The trade-off is +# that a higher floor reintroduces some early seed-divergence, so it costs +# some of warmup's CV-reduction benefit; keep `start` << base to stay in the +# low-CV regime. +# +# HOW TO ENABLE / DISABLE +# Controlled by the $LR_WARMUP_START_LR environment variable (read at startup): +# - DEFAULT 0.0: classic 0 -> base ramp; byte-identical to plain warmup. +# - NONZERO: export LR_WARMUP_START_LR= (must be < base LR) +# e.g. at base=1e-7: 1.25e-8 (= base/8) 2.5e-8 (= base/4) +# No effect unless $LR_WARMUP_STEPS > 0 (there is no ramp to lift a floor on). +streaming_train_eval_loop.lr_warmup_start_lr = @lrws/env_float() +lrws/env_float.key = "LR_WARMUP_START_LR" +lrws/env_float.default = 0.0 # 0.0 = classic 0 -> base ramp (baseline-safe) + +# Data root: resolved at runtime from $DLRM_DATA_PATH if set, else the literal +# below. Used by both make_train_test_dataloaders and get_dataset. +# Scoped (`data/env_path`) so this binding doesn't collide with the RUN_NAME +# env_path binding below — every distinct env_path() call site needs its own +# scope or the later `env_path.key=...` overrides earlier ones. +DATA_PATH = @data/env_path() +data/env_path.key = "DLRM_DATA_PATH" +data/env_path.default = "/apps/chcai/dlrm_data" + +# Shared train:eval split: fraction of USERS used for training; the remaining +# (1 - this) fraction are held out as a fixed eval set and NEVER trained. +# Bound to BOTH the static train-eval path (make_train_test_dataloaders, a +# positional split) and the streaming path (get_dataset, an explicit by-user +# hash split), so one value configures the holdout in either mode. +# 1.0 = no holdout (legacy streaming behavior). Override via $TRAIN_SPLIT_PERCENTAGE. +# Default 1.0: all users are trained AND evaluated (full-coverage eval), matching +# the alleval/qa2a production runs; set <1.0 (e.g. 0.90) for a clean held-out cohort. +TRAIN_SPLIT_PERCENTAGE = @tsp/env_float() +tsp/env_float.key = "TRAIN_SPLIT_PERCENTAGE" +tsp/env_float.default = 1.0 + +# dataloader configs +make_train_test_dataloaders.batch_size = %batch_size +make_train_test_dataloaders.eval_batch_size = 1024 +make_train_test_dataloaders.dataset_type = %dataset +make_train_test_dataloaders.train_split_percentage = %TRAIN_SPLIT_PERCENTAGE +make_train_test_dataloaders.new_path_prefix = %DATA_PATH +make_train_test_dataloaders.num_workers = %num_workers +make_train_test_dataloaders.prefetch_factor = %prefetch_factor +make_train_test_dataloaders.num_blocks = 1 + +# embedding planner: per-rank HBM ceiling the torchrec sharder targets. +# Override via $HBM_CAP_GB (e.g. lower to 150 to force more CW sharding). +make_optimizer_and_shard.hbm_cap_gb = @env_int() +env_int.key = "HBM_CAP_GB" +env_int.default = 260 + +# Embedding table placement. Global default applied to EVERY table: +# auto -> no constraint; planner decides from the HBM cap (default) +# hbm -> FUSED (resident in GPU HBM) +# uvm -> FUSED_UVM (host DDR via UVM, no HBM cache) +# uvm_caching -> FUSED_UVM_CACHING (host DDR + HBM cache) +# "force HBM" for all tables = set this to "hbm" (or export EMB_PLACEMENT=hbm). +make_optimizer_and_shard.embedding_placement = @emp/env_str() +emp/env_str.key = "EMB_PLACEMENT" +emp/env_str.default = "auto" + +# Per-table placement overrides (win over the global default above). Configure +# each table independently here as a gin dict; allowed values per table: +# "hbm" | "uvm" | "uvm_caching" | "auto" (unlisted tables use the global +# default above). merge=True means a per-run env override LAYERS ON TOP of this +# dict per key (tweak one table at launch, keep the rest), e.g.: +# EMB_PLACEMENT_OVERRIDES="uid=hbm" # only retargets uid; others stay as below +make_optimizer_and_shard.embedding_placement_overrides = @env_str_map() +env_str_map.key = "EMB_PLACEMENT_OVERRIDES" +env_str_map.merge = True +env_str_map.default = {} + +# Per-table SHARDING-TYPE overrides (shard layout, orthogonal to placement above). +# Absent/"auto" tables keep the planner's choice (ROW_WISE for the large yambda +# tables). Allowed per table: "row_wise"|"column_wise"|"table_wise"| +# "table_row_wise"|"table_column_wise"|"data_parallel"|"auto" (aliases rw/cw/tw/ +# twrw). DEFAULT OFF ({}) -> plan is byte-identical to the legacy all-ROW_WISE +# path. Opt in per run via env, e.g.: +# EMB_SHARDING_OVERRIDES="album_id=column_wise,artist_id=column_wise" +# WHY: ROW_WISE routes every lookup of row r to the single owner rank, so a few +# hot IDs concentrate the embedding all-to-all onto one rank and OOM it (the +# yambda-5b skew hang; album_id ~2.8x, artist_id ~1.3x per-rank load). COLUMN_WISE +# splits the table by embedding dim (every rank holds all rows, dim/world cols), +# so the a2a load is balanced by RANK regardless of which IDs are hot — removing +# the value-skew OOM — at identical per-rank table bytes. Convert only the skewed +# high-volume tables (album_id, artist_id); leave the balanced, highest-volume +# item_id and the tiny length-1 contextual/cross tables on ROW_WISE. +# WHY THIS GETS WORSE AT LARGER GLOBAL BATCH: the ROW_WISE a2a input buffer on the +# owner rank is sized by how many times its hot IDs appear across the WHOLE global +# batch, so that transient scales ~linearly with global batch size (here 32 ranks +# x 1024 = 32768, each carrying ~4096-token UIH sequences -> tens of millions of +# lookups/step, heavily re-hitting the same few popular albums/artists). Doubling +# the global batch ~doubles the hot-rank burst while every other rank stays idle, +# which is exactly what tipped GPU5/GPU3 from a saturated steady state (~208-238 +# GiB) over 288 GiB at window ~248. COLUMN_WISE makes each rank receive dim/world +# of EVERY lookup, so the per-rank a2a volume is ~global_batch/world (balanced) +# and grows with world size, not with which IDs are hot -> it scales cleanly as +# you push global batch / add ranks, instead of piling the growth onto one shard. +# Example (the yambda-5b 4-node run, global batch 32768, that hit the skew OOM): +# EMB_SHARDING_OVERRIDES="album_id=column_wise,artist_id=column_wise" +# validated by the CW smoke: reshard-loaded the ROW_WISE ckpt cleanly, window_auc +# stayed ~0.78-0.80, and the hot rank sat at ~58% (~120 GiB free) THROUGH the old +# OOM window (vs 16 GiB free right before the ROW_WISE crash). +# NOTE: changing the sharding plan changes the on-disk shard layout; a checkpoint +# written under a different plan must be resharded/validated on load. +make_optimizer_and_shard.embedding_sharding_overrides = @esh/env_str_map() +esh/env_str_map.key = "EMB_SHARDING_OVERRIDES" +esh/env_str_map.merge = True +esh/env_str_map.default = {} + +# Sparse embedding all-to-all wire precision. The embedding shuffle is the +# dominant, bandwidth-bound (esp. multi-node) collective; quantizing it via +# TorchRec QCommsConfig halves (bf16/fp16, both 2 bytes) the wire volume. +# Forward and backward are set independently (each: "fp32" | "bf16" | "fp16"). +# Both "fp32" = off (numerically identical to baseline trunk). +# Default is now fp16/fp16: an A/B run vs the fp32 baseline matched window AUC to +# within fixed-seed noise (mean Δ≈-1e-6, max|Δ|≈5e-5, r=1.0 over 0-54.5% data) +# while cutting the embedding-a2a wire volume in half (~6% end-to-end speedup). +# TorchRec golden_training suggests fwd=fp16 / bwd=bf16 (bf16's wider exponent +# hedges gradient overflow), but on yambda-5b the grads stay well in fp16 range +# (grad-clip=1.0, lr=1e-7), so fp16/fp16 is convergence-neutral here. +# Override via $SPARSE_A2A_FWD / $SPARSE_A2A_BWD (e.g. set both "fp32" to disable). +make_optimizer_and_shard.sparse_a2a_forward_precision = @saaf/env_str() +saaf/env_str.key = "SPARSE_A2A_FWD" +saaf/env_str.default = "fp16" +make_optimizer_and_shard.sparse_a2a_backward_precision = @saab/env_str() +saab/env_str.key = "SPARSE_A2A_BWD" +saab/env_str.default = "fp16" + +# ============================================================================= +# RUNTIME PATCHES / MONKEYPATCHES +# ----------------------------------------------------------------------------- +# Third-party kernels we override AT RUNTIME (no fork of the dependency). Each +# knob here is a kill switch for one such patch: ON by default (it fixes a real +# bug we hit), but flippable per-run so we can A/B the patch, reproduce the +# original failure, or fall back to stock if a future dependency version makes +# the patch unnecessary or incorrect. Patches are applied during model +# build/shard (make_optimizer_and_shard), before any training step. +# ============================================================================= +# +# --- qcomm_lowmem_clamp_cast: low-memory fbgemm fp16/bf16 quant codec --------- +# WHAT IT PATCHES +# fbgemm_gpu's embedding-a2a quantizer, fp32_to_fp16_with_clamp (and the bf16 +# variant), which the TorchRec qcomm codec calls to pack the sparse +# embedding all-to-all payload onto the wire. Stock implementation is: +# torch.clamp(tensor, HALF_MIN, HALF_MAX).half() +# torch.clamp() materializes a SECOND full-size fp32 tensor (same numel as the +# input) BEFORE the cast, so the transient peak is +# input(fp32) + clamp_temp(fp32) + output(fp16) ~= 2.5x the input. +# The patch reorders to an in-place, allocation-free-equivalent: +# tensor.half().clamp_(HALF_MIN, HALF_MAX) +# i.e. cast FIRST (only the fp16 output is allocated), then clamp IN PLACE — +# dropping the full-size fp32 clamp temp and cutting the transient peak by the +# size of the input tensor. +# +# WHY IT'S NEEDED (the bug it fixes) +# Embeddings are ROW-WISE sharded: every lookup of row r routes to the single +# rank that owns r. On a data-skewed batch a few extremely hot IDs send a +# disproportionate share of the global lookups to ONE owner rank, so that +# rank's a2a input tensor balloons. With the stock codec the extra fp32 clamp +# temp on that rank reached ~81.5 GiB, which OOM'd the rank mid-forward. The +# OOM'd rank then dropped out of the collective while its peers blocked forever +# in the embedding all-to-all -> ~30-min NCCL watchdog timeout -> the whole job +# SIGABRTs. This was a DETERMINISTIC hang (same window/step every run; e.g. +# yambda-5b 4-node fp16 a2a hit it at window 235 / global step 43621). +# See generative_recommenders/dlrm_v3/train/utils.py +# (_patch_fbgemm_lowmem_clamp_cast) for the full diagnosis + flight-recorder +# evidence. +# +# CORRECTNESS / PERF +# Numerically IDENTICAL (bit-for-bit) to stock: the single fp32->fp16 cast is +# the only rounding step in both orders; an fp32 value above HALF_MAX casts to +# +inf which clamp_ maps back to HALF_MAX, and NaNs pass through unchanged. +# In-place is safe because the codec encode() runs inside the qcomm autograd +# Function.forward with grad disabled (no graph to corrupt). No throughput +# regression (measured step time within run-to-run noise; it does strictly +# LESS memory traffic + allocation than stock). +# +# WHEN TO TURN OFF (set 0): only to reproduce the original OOM/hang for +# debugging, or to revalidate against stock after an fbgemm_gpu upgrade. +# Override via $QCOMM_LOWMEM_CODEC. Only takes effect when the embedding a2a is +# actually quantized (SPARSE_A2A_FWD/BWD not both fp32); with fp32 a2a there is +# no codec to patch and this knob is a no-op. +make_optimizer_and_shard.qcomm_lowmem_clamp_cast = @qlcc/env_int() +qlcc/env_int.key = "QCOMM_LOWMEM_CODEC" +qlcc/env_int.default = 1 + +# --- ec_index_dedup: in-batch embedding index deduplication ------------------ +# Enables TorchRec's use_index_dedup on the sequence EmbeddingCollectionSharder. +# Before the sparse input all-to-all, duplicate ids within a batch are collapsed +# to unique rows (torch.ops.fbgemm.jagged_unique_indices), the embedding lookup +# runs once per unique id, then results scatter back to their original positions +# via reverse_indices. For a NON-POOLED EmbeddingCollection this reconstruction +# is exact, so it is numerically LOSSLESS (unlike the fp16 a2a it does not move +# AUC/NE) — it only cuts (a) the sparse input all-to-all element COUNT and (b) +# the number of rows the fused TBE kernel gathers per step. +# +# WHY IT HELPS HERE: the default streaming order is user-major +# (STREAMING_SHUFFLE_FRACTION=0), so consecutive anchors are the same user's UIH +# re-reading the same item/artist/album ids -> very high in-batch duplicate rate. +# Measured via DIAG_UNIQUE_EMB on this config: the big UIH sequence tables are +# ~98% duplicate WITHIN a batch (uih.item_id ~2.4% unique over 2.69M ids/batch, +# uih.album_id ~1.5%, uih.artist_id ~0.6%), so collapsing to unique rows removes +# the bulk of the lookup + a2a volume. Dedup COMPOSES with the fp16 qcomm a2a +# (dedup cuts element count, qcomm cuts bytes/element) and the CW skew fix. +# +# MEASURED A/B (this exact config; fixed SEED=1; qcomm fp16 fwd+bwd; 200-step +# train-only; c-series MI350 nodes). Steady-state (steps>=60) mean step time and +# global throughput, dedup OFF -> ON: +# * 1 node (WORLD_SIZE=8): step_ms 941.6 -> 805.7 (-14.4%, +16.9% sps), +# MFU 21.3% -> 24.7% +# * 2 node (WORLD_SIZE=16): step_ms 1108.0 -> 818.0 (-26.2%, +37.6% sps), +# MFU 18.1% -> 24.9% +# The win GROWS with node count because at >1 node dedup also shrinks the +# cross-node embedding all-to-all — the dominant bandwidth-bound multi-node +# collective. It nearly erases the multi-node comm penalty: OFF weak-scales +# 21.3%->18.1% MFU going 1->2 node, ON holds ~24.7%->24.9%. +# +# NUMERICAL NEUTRALITY (why default-on is safe): with a fixed seed the OFF and +# ON runs produced IDENTICAL metrics, confirming the reconstruction is exact and +# training is bit-for-bit unaffected: +# * 1 node @ step41: train_loss 0.13778 == 0.13778; window_ne/lifetime_ne +# 1.0338 == 1.0338; window_auc 0.4947 == 0.4947 +# * 2 node @ step41: train_loss 0.13811 == 0.13811 +# (Contrast the fp16 a2a, which is lossy-but-tolerable; this knob moves nothing.) +# +# The dedup buffers are persistent=False -> NO change to the on-disk shard layout +# and safe to toggle on an existing checkpoint (no reshard). Applied as a final +# pass over the sharder list so it covers BOTH the qcomm-on (default) and +# qcomm-off (SPARSE_A2A_*=fp32) construction paths. +# +# DEFAULT ON (1): lossless + a large, node-count-scaling speedup on this config. +# The payoff shrinks toward neutral/slightly-negative only as batch diversity +# rises (high STREAMING_SHUFFLE_FRACTION -> fewer in-batch dups, so the +# jagged_unique_indices + scatter overhead stops paying for itself); set +# EC_INDEX_DEDUP=0 for such a high-shuffle run. Override per-run via $EC_INDEX_DEDUP. +make_optimizer_and_shard.ec_index_dedup = @ecid/env_int() +ecid/env_int.key = "EC_INDEX_DEDUP" +ecid/env_int.default = 1 + +get_dataset.name = %dataset +get_dataset.new_path_prefix = %DATA_PATH +# Total user-interaction-history (UIH) budget per sample, distributed evenly +# across 3 behaviour pools (listen+ / like / skip) at L//3 events each. +# Per-sample sequence the model sees = +# 3 × (L // 3) + 8 contextual + 1 candidate +# Choosing 4086 makes 3 × 1362 + 9 = 4095, the largest value that fits +# get_hstu_configs.max_seq_len = 4096 with no dataset-side truncation (the +# 4k analog of the previous 2039/2048 shape, where 3×679+9=2046 ≤ 2048). +# Larger L overflows the budget; the dataset truncates UIH events to fit. +# Note: like events are only 1.9% of the yambda corpus and max user lifetime +# is ~28k events, so the like pool fills to ~105 events per anchor on +# average (not 1362) — TRITON's jagged attention skips the unfilled slots, +# so the under-fill costs sequence budget but not GPU compute. +# Cache is keyed by L on disk under /hstu_cache_L/; +# switching L reuses an existing cache or builds a new one. Override via +# $HISTORY_LENGTH (default 4086 = the 4k-no-truncation shape; use 2039 with +# MAX_SEQ_LEN=2048 to reuse the previous 2k single-task cache). +get_dataset.history_length = @hl/env_int() +hl/env_int.key = "HISTORY_LENGTH" +hl/env_int.default = 4086 + +# UIH construction strategy — how a user's prior events become the sequence the +# model attends over. Override via $HISTORY_STRATEGY. Both strategies scan the +# last `scan_window` (20000) events before the anchor and consider ONLY the 3 +# behavior pools (listen+ / like / skip); dislike/unlike/undislike are excluded +# (no action-weight bit), so the model's action_weights=[1,2,4] are unchanged. +# The two differ only in HOW they pick events out of that scan window: +# +# "interleaved" (default): +# - Budget = equal per-pool quota of HISTORY_LENGTH//3 (=1362) events; take +# the last 1362 of EACH pool independently, then merge + re-sort by time. +# - HISTORY_LENGTH is thus a PER-POOL cap (nominal total = 3 * L//3). +# - Consequence: likes are only ~1.9% of the corpus, so the like pool +# under-fills (~105 of its 1362 slots) and that budget is NOT reallocated +# — the sequence comes up short. Measured: ~2663 events effective, ~4.0% +# of them likes (i.e. likes OVER-represented vs their natural rate). +# - Use when you want guaranteed like exposure per sample (class-balanced UIH). +# +# "last_n": +# - Budget = a single pool-agnostic cap: take the last HISTORY_LENGTH events +# of ANY pool type, already chronological (no per-pool split, no re-sort). +# - HISTORY_LENGTH is thus the LITERAL TOTAL UIH cap (not a per-pool L//3). +# - Consequence: the sequence fills to ~HISTORY_LENGTH (the only limit is how +# many pooled events exist in the scan window), so effective length is +# higher and the like share falls to its natural rate. Measured: ~4085 +# events effective (~1.5x interleaved), ~1.2% of them likes. +# - Trade-off: more recent listen+/skip context (longer effective sequence) +# at the cost of fewer likes; ~1.4x eval/step compute (more non-skipped +# jagged-attention positions). Keep HISTORY_LENGTH=4086 to fill the 4k +# model budget. +# +# Both strategies are strategy-INDEPENDENT on disk: the gather runs at +# sample-build time, so switching reuses the SAME hstu_cache_L/ +# and positions cache — no rebuild needed. +get_dataset.history_strategy = @hs/env_str() +hs/env_str.key = "HISTORY_STRATEGY" +hs/env_str.default = "interleaved" + +# Anchor-eligibility floor: a LISTEN event becomes a trainable/eval anchor once +# the user has >= MIN_HISTORY prior events. Decoupled from history_length (which +# is only the gather/truncation cap) — jagged attention handles short UIH, so we +# no longer need a full history_length of context to include a sample. The legacy +# behavior (require a full history_length, which dropped ~60% of users) is +# MIN_HISTORY=4086; the default below (0) includes ~all users AND their +# cold-start first event (zero prior context). Positions/anchor-ts caches are +# keyed by (L, MIN_HISTORY) so floors don't collide. Override via $MIN_HISTORY. +get_dataset.min_history = @mh/env_int() +mh/env_int.key = "MIN_HISTORY" +mh/env_int.default = 4086 + +# Model-side attention budget. Dataset truncates UIH to fit this value if +# `history_length + contextual + candidate` would overflow. Override via +# $MAX_SEQ_LEN (default 4096, the 4k-no-truncation shape paired with +# HISTORY_LENGTH=4086: 3*1362+9=4095 ≤ 4096). Set MAX_SEQ_LEN=2048 with +# HISTORY_LENGTH=2039 for the previous 2k production single-task shape. +get_hstu_configs.max_seq_len = @msl/env_int() +msl/env_int.key = "MAX_SEQ_LEN" +msl/env_int.default = 4096 + +# HSTU transformer depth (number of attention layers). Default 3. +# Override per-run via $HSTU_NUM_LAYERS. NOTE: changing depth changes the model +# shape, so a run with a new depth MUST use a FRESH CKPT_PATH (incompatible with +# checkpoints of a different depth). Resolved in the full gin parse +# (get_hstu_configs is not registered during the early skip_unknown parse), so +# the @env_int reference is skipped on the first pass — same safe path as the LR knobs. +get_hstu_configs.hstu_attn_num_layers = @nl/env_int() +nl/env_int.key = "HSTU_NUM_LAYERS" +nl/env_int.default = 3 + +# --- streaming (temporal-order) training ------------------------------------- +# Only consumed under `--mode streaming-train-eval`; the default train-eval +# path above is unaffected. Trains time window T then evals window T+1, +# advancing forward in wall-clock time (no future leakage). Window size is the +# temporal-ordering granularity knob (default 1 day). num_train_ts is clamped +# to the dataset's available window count at runtime; override via $NUM_TRAIN_TS. +get_dataset.streaming_window_seconds = 86400 +get_dataset.streaming_sort_within_window = False +# In-window shuffle to break user-major batching (consecutive sliding-window +# anchors otherwise come from the same few users -> few unique embedding reads). +# Diversity dial in [0,1] -- the AGREED, config-invariant benchmark knob. Maps to +# a within-segment shuffle with K = round(fraction * per-window train-anchor +# count): 0 = off (user-major, page-local mmap scans), 1 = full per-element +# shuffle (max diversity), intermediate = interpolation. Same fraction => same +# diversity regardless of world_size / #nodes / batch_size. +# +# Default 0.0 (no shuffle -> user-major, page-local mmap scans) so the standard +# run matches the production streaming order. Together with the fixed seed below +# the in-window order is fully DETERMINISTIC and identical across runs/resumes. +# Override per-run via $STREAMING_SHUFFLE_FRACTION (e.g. 1.0 for a full +# per-element shuffle / max diversity, 0.03 for the diversity/locality sweet spot). +streaming_shuffle_fraction = 0.0 +get_dataset.streaming_shuffle_fraction = @ssf/env_float() +ssf/env_float.key = "STREAMING_SHUFFLE_FRACTION" +ssf/env_float.default = %streaming_shuffle_fraction +# Fixed shuffle seed -> reproducible permutation. Exposed as a knob; override via +# $STREAMING_SHUFFLE_SEED only if you deliberately want a different draw. +get_dataset.streaming_shuffle_seed = @ssfseed/env_int() +ssfseed/env_int.key = "STREAMING_SHUFFLE_SEED" +ssfseed/env_int.default = 0 +# User-level train:eval holdout for the streaming path. With tsp<1.0, the top +# (1 - tsp) fraction of users (by a deterministic hash of uid+split_salt) are +# held out as a FIXED eval set and never trained -> no temporal/user leakage, +# stable comparable eval curve, bounded eval time. split_salt lets you draw a +# different holdout without changing the ratio. Override salt via $SPLIT_SALT. +get_dataset.train_split_percentage = %TRAIN_SPLIT_PERCENTAGE +get_dataset.split_salt = @ssalt/env_int() +ssalt/env_int.key = "SPLIT_SALT" +ssalt/env_int.default = 0 +make_streaming_dataloader.batch_size = %batch_size +make_streaming_dataloader.num_workers = %num_workers +make_streaming_dataloader.prefetch_factor = %prefetch_factor +make_persistent_streaming_dataloader.batch_size = %batch_size +make_persistent_streaming_dataloader.num_workers = %num_workers +make_persistent_streaming_dataloader.prefetch_factor = %prefetch_factor +streaming_train_eval_loop.num_train_ts = @nts/env_int() +nts/env_int.key = "NUM_TRAIN_TS" +# 299 daily windows -> with start_ts=0 the run sweeps ts=0..298, the full corpus +# (matches the long fp16 e2e run). Clamped to the dataset's available window +# count at runtime. Override via $NUM_TRAIN_TS. +nts/env_int.default = 299 +# Anchors need >= history_length prior events, so the first ~130 daily windows +# are near-empty warm-up. Default start_ts=0 trains the full corpus from the +# start (matches the canonical fp16 run); set $START_TS=150 to skip warm-up and +# begin at a dense window. Override via $START_TS. +streaming_train_eval_loop.start_ts = @sts/env_int() +sts/env_int.key = "START_TS" +sts/env_int.default = 0 +# Per-step metric logging cadence. Default 50 (one compute_and_log GPU->CPU +# sync per 50 batches). The streaming-resume test sets METRIC_LOG_FREQ=1 so +# every step emits a parseable "Step N metrics" line for trajectory comparison. +streaming_train_eval_loop.metric_log_frequency = @mlf/env_int() +mlf/env_int.key = "METRIC_LOG_FREQ" +mlf/env_int.default = 50 +# Interim eval logging: when >0, log partial-holdout metrics every +# METRIC_LOG_FREQ batches DURING an eval pass (debug/progress only). The durable +# .metrics.jsonl, early-stop, and MLPerf EVAL_ACCURACY all use ONLY the +# end-of-pass whole-holdout compute(), so these interim calls are redundant AUC +# reductions (a BinnedCumulativeAUC histogram all-reduce + GPU->CPU sync per +# call, x window_ and lifetime_ sets). 0 (default) = OFF -> skip them. +# +# *** AUC-METRIC NEUTRAL — toggling this NEVER changes the reported eval AUC. *** +# WHY (by construction): every eval batch's predictions/labels are folded into +# the metric state via metric_logger.update() UNCONDITIONALLY, once per batch, +# regardless of this flag. The reported eval AUC is the SINGLE end-of-pass +# metric_logger.compute(mode="eval") over the whole holdout, which is a pure +# function of that accumulated state (BinnedCumulativeAUC integrates its score +# histogram; compute() does NOT mutate the histogram). The interim +# compute_and_log() calls this flag gates only READ that same state to print a +# running partial-holdout number — they cannot alter what update() accumulated, +# so the final AUC is identical whether they fire 0 or N times. This is why the +# flag is safe to leave OFF by default (pure logging removal, not a metric change). +# VERIFIED e2e (SEED=1, same node, 2 windows, interim actually firing when ON): +# OFF-vs-ON max window/lifetime AUC |diff| = 2.4e-6 +# OFF-vs-OFF control (identical runs) max AUC |diff| = 3.3e-6 +# i.e. the flag's effect on AUC is BELOW the run-to-run GPU FP nondeterminism +# floor (the ~1e-6 residual is nondeterministic training kernels, not this flag). +# Override via $EVAL_INTERIM_LOG. +streaming_train_eval_loop.eval_interim_metrics = @eil/env_int() +eil/env_int.key = "EVAL_INTERIM_LOG" +eil/env_int.default = 0 +# MLPerf train_loss event cadence (global train steps), INDEPENDENT of +# METRIC_LOG_FREQ above. 0 (default) = fall back to METRIC_LOG_FREQ, preserving +# the prior coupled behavior. Set $MLPERF_TRAIN_LOSS_LOG_FREQ>0 to log the MLPerf +# train_loss event at a different rate than the console/TB metrics. Disable the +# whole MLPerf stream with $MLPERF_LOGGING=0. +streaming_train_eval_loop.mlperf_train_loss_log_frequency = @mltlf/env_int() +mltlf/env_int.key = "MLPERF_TRAIN_LOSS_LOG_FREQ" +mltlf/env_int.default = 0 +# Diagnostic: log per-batch unique/total embedding-id counts on logged steps +# (rank 0). Quantifies the user-major batching redundancy and the realized +# diversity from get_dataset.streaming_shuffle_fraction. Off; set $DIAG_UNIQUE_EMB=1. +streaming_train_eval_loop.streaming_diag_unique_emb = @due/env_int() +due/env_int.key = "DIAG_UNIQUE_EMB" +due/env_int.default = 0 +# Chrome trace capture: reuses the shared Profiler.* bindings below (5-step +# window at step 52). The streaming step counter advances across train+eval +# batches, so step 52 lands in the first (train) window's compute. Off by +# default to avoid profiler overhead in production runs; set $OUTPUT_TRACE=1. +streaming_train_eval_loop.output_trace = @ot/env_int() +ot/env_int.key = "OUTPUT_TRACE" +ot/env_int.default = 0 +# Reuse one DataLoader (persistent workers) across windows instead of respawning +# per window. Skip eval to isolate window-reset cost. Override via env. +streaming_train_eval_loop.persistent_loader = @pl/env_int() +pl/env_int.key = "PERSISTENT_LOADER" +pl/env_int.default = 1 +# ---- Eval cadence: choose EXACTLY ONE of the two knobs below ---------------- +# The streaming loop can decide WHEN to run the full-holdout eval pass in one of +# two ways. They are MUTUALLY EXCLUSIVE — enabling the data-fraction cadence +# (EVAL_EVERY_DATA_PCT>0) requires EVAL_EVERY_N_WINDOWS=0; setting both >0 raises +# a ValueError at startup. The final end-of-run eval always runs in either mode. +# +# (1) PER-WINDOW cadence (EVAL_EVERY_N_WINDOWS). +# Full-holdout eval cadence (single knob; replaces the old EVAL_EACH_WINDOW +# on/off switch). 0 (default) = per-window cadence OFF -> defer to the +# data-fraction cadence below (EVAL_EVERY_DATA_PCT); if that is also 0, eval is +# disabled entirely (train-only, e.g. perf benchmarking or the resume test; the +# eval dataloader isn't even built). 1 = eval after every window. N>1 (e.g. 5 +# via $EVAL_EVERY_N_WINDOWS) = eval every Nth window (and always the final one) +# to amortize the cost of consuming the full next-day eval window. The cadence is +# anchored to the absolute ts grid so eval points stay stable across a mid-run +# resume. +# NOTE: each daily window has a DIFFERENT number of training samples, so a +# per-window cadence produces eval points that are UNEVENLY spaced in terms of +# how much data was trained between them. This is why the data-fraction cadence +# below is now the default; enable this per-window knob only if you specifically +# want eval anchored to the daily window grid. +streaming_train_eval_loop.eval_every_n_windows = @evn/env_int() +evn/env_int.key = "EVAL_EVERY_N_WINDOWS" +evn/env_int.default = 0 +# +# (2) DATA-FRACTION cadence (EVAL_EVERY_DATA_PCT). +# Run the full-holdout eval every time the run has trained this FRACTION of the +# run's TOTAL training data, so eval points are EVENLY spaced by data volume +# (compute), independent of how many samples each daily window happens to hold. +# This is the fix for the per-window cadence's uneven spacing noted above. +# value semantics (it is a fraction in (0, 1], NOT a percent number): +# 0.0 = OFF -> fall back to the per-window EVAL_EVERY_N_WINDOWS; +# if that is also 0, eval is disabled entirely (train-only). +# 0.005 (default)= eval every 0.5% of the data -> ~200 eval points total. +# 0.01 = eval every 1% of the data -> ~100 eval points total. +# 0.05 = eval every 5% of the data -> ~20 eval points total. +# 0.10 = eval every 10% of the data -> ~10 eval points total. +# How "fraction of data" becomes an eval trigger: at startup the fraction is +# converted ONCE into a global train-step interval +# eval_interval_steps = round(pct * total_train_anchors / (batch_size * world_size)) +# where total_train_anchors is counted over the ORIGINAL requested window range +# [start_ts, start_ts+num_train_ts). Eval then fires whenever the monotonic +# train global_step crosses a multiple of that interval (it can fire MID-WINDOW, +# i.e. partway through a daily window, and across window boundaries). Because the +# interval is computed over the original range and global_step is +# checkpoint-restored, the eval grid is identical on a cold start and on every +# resume. Each eval record's label carries "@step=" so the +# trajectory can be plotted against data volume. Override via $EVAL_EVERY_DATA_PCT. +streaming_train_eval_loop.eval_every_data_pct = @edp/env_float() +edp/env_float.key = "EVAL_EVERY_DATA_PCT" +edp/env_float.default = 0.001 +# ---- SKIP_EVAL: warm-up eval skipping (early-run overhead cut) -------------- +# WHAT IT DOES +# Suppresses the FIRST N *scheduled/periodic* full-holdout eval passes. Early +# in a run the model sits far below the AUC target and those first eval points +# carry little signal, yet each full-holdout pass is a real cost (a forward +# sweep over the whole held-out set + its metric reduction). SKIP_EVAL lets you +# pay that cost only once training has advanced enough to matter. +# N = $SKIP_EVAL (default 10 -> skip the first 10 scheduled eval passes; +# set 0 to eval from the very first scheduled point.) +# +# HOW THE "FIRST N" ARE COUNTED (per cadence; only the active cadence applies) +# Eval boundaries are numbered on the SAME grid the cadence already uses, and +# a boundary is skipped while its index falls in the first N: +# * DATA-FRACTION cadence (EVAL_EVERY_DATA_PCT>0): a boundary fires when the +# global train step hits k*eval_interval_steps (k=1,2,3,...). Its 1-based +# ordinal is k = global_step / eval_interval_steps; the boundary is SKIPPED +# while k <= N. So eval begins at the (N+1)th data-fraction point, i.e. after +# ~ (N+1) * EVAL_EVERY_DATA_PCT of the data has been trained. +# * PER-WINDOW cadence (EVAL_EVERY_N_WINDOWS>0): the eval-eligible windows are +# numbered on the absolute-ts grid as ordinal = +# (train_ts - eval_anchor_ts) / eval_every_n_windows (0-based); a window is +# SKIPPED while ordinal < N. +# +# RESUME SAFETY +# The ordinal is derived purely from the checkpoint-restored global_step +# (data-fraction) or the absolute ts grid (per-window) -- never from a +# per-process counter -- so the skipped set is IDENTICAL on a cold start and on +# every mid-run resume (a resume cannot re-skip points it already trained past, +# nor start eval'ing points an earlier attempt skipped). +# +# WHAT IS NEVER SKIPPED +# The end-of-run FINAL eval always runs (it is the definitive convergence +# number), and early-stop / MLPerf EVAL_ACCURACY still fire normally once eval +# starts at the (N+1)th point. SKIP_EVAL only removes early *periodic* passes. +# +# EXAMPLE (verified e2e; per-window cadence, START_TS=150, NUM_TRAIN_TS=4, +# EVAL_EVERY_N_WINDOWS=1 -> eligible eval points ts=150,151,152,153 + final): +# SKIP_EVAL=0 -> eval at ts=150,151,152,153,final (all points) +# SKIP_EVAL=2 -> eval at ts=152,153,final (first two dropped) +# Override via $SKIP_EVAL. +streaming_train_eval_loop.skip_eval = @se/env_int() +se/env_int.key = "SKIP_EVAL" +se/env_int.default = 10 +# Double-buffer windows: prepare the next window (index mask + first-batch +# prefetch) in a background thread during the current window's compute, hiding +# the per-window reset. Needs persistent_loader=1. Override via env. +streaming_train_eval_loop.double_buffer = @db/env_int() +db/env_int.key = "DOUBLE_BUFFER" +db/env_int.default = 1 +# Fixed eval-holdout window range (held-out users' anchors over these windows +# form the eval set evaluated at EVERY eval step). Default 299 = the window just +# past the ts=0..298 training sweep (matches the canonical fp16 run). Set +# EVAL_HOLDOUT_TS<0 to instead resolve at runtime to start_ts+num_train_ts (also +# stable across resume). EVAL_HOLDOUT_NUM_WINDOWS widens the eval span. +streaming_train_eval_loop.eval_holdout_ts = @eht/env_int() +eht/env_int.key = "EVAL_HOLDOUT_TS" +eht/env_int.default = 299 +streaming_train_eval_loop.eval_holdout_num_windows = @ehnw/env_int() +ehnw/env_int.key = "EVAL_HOLDOUT_NUM_WINDOWS" +ehnw/env_int.default = 1 +# num_train_batches / num_eval_batches unset => consume each full window. +# Set them (e.g. via gin) to cap per-window steps for short experiments. + +# Default (non-streaming) train-eval loop variables; used unless +# `--mode streaming-train-eval` selects the temporal-order path configured above. +train_eval_loop.num_epochs = 1 +train_eval_loop.metric_log_frequency = 50 +train_eval_loop.eval_frequency = 500 +train_eval_loop.num_eval_batches = 200 +train_eval_loop.checkpoint_frequency = 1000000000 # disable mid-training checkpoints (disk-full guard) +train_eval_loop.output_trace = True +# 3-stage TorchRec pipeline: overlaps the embedding all-to-all (the dominant +# exposed-comm bottleneck) with dense fwd/bwd. Set False to fall back to the +# sequential fwd/bwd loop. +train_eval_loop.use_pipeline = False + +# Run name → recommendation_v4/results// (override via $RUN_NAME env). +RUN_NAME = @run/env_path() +run/env_path.key = "RUN_NAME" +run/env_path.default = "default" +run_results_dir.run_name = %RUN_NAME + +# profiler: capture a 5-step window starting at step 52, every rank. +# `trim_warmup = True` (default) post-filters the chrome trace so only the 5 +# `active` steps appear; the 1 WARMUP step still runs (lets CUPTI/ROCprof +# settle) but its events are dropped from the output file. +Profiler.trace_steps = [52] +Profiler.warmup = 1 +Profiler.active = 5 +Profiler.trace_dir = @run_results_dir() + +# logger variables +# TensorBoard event dir. DISABLED BY DEFAULT (empty path): the shared-NFS +# tfevents writer is the only metrics sink whose I/O error is uncaught, and it +# repeatedly crashed trainers on transient /apps `Errno 121` (Remote I/O) hiccups. +# Nothing we consume reads TensorBoard — eval-window AUCs come from the durable +# `.metrics.jsonl` sink (try/except-guarded) and the text run log. An empty +# path makes MetricsLogger install a no-op writer (see _NoOpSummaryWriter). +# Re-enable for a run by setting $TENSORBOARD_LOG_PATH to a non-empty dir. +MetricsLogger.tensorboard_log_path = @tbp/env_path() +tbp/env_path.key = "TENSORBOARD_LOG_PATH" +tbp/env_path.default = "" +MetricsLogger.world_size = 8 +# MLPerf convergence target: run stops when the selected eval AUC reaches it. +# Default 1.0 is unreachable, so the run trains through all windows (no early +# stop) out of the box; set $AUC_THRESHOLD=0.80275 for the MLPerf target. +MetricsLogger.auc_threshold = @at/env_float() +at/env_float.key = "AUC_THRESHOLD" +at/env_float.default = 1.0 +# EVAL_ACCURACY + early-stop are driven by per-window AUC (window_auc) vs the +# threshold above. +# Lifetime-AUC backend, selectable independently for the train cumulative AUC and +# the eval cumulative ("lifetime_*") AUC. Both default to "binned": +# "binned" = BinnedCumulativeAUC: exact-cumulative AUC from an O(bins) score +# histogram (additive all-reduce, memory independent of #samples/#windows). +# "capped" = LifetimeAUCMetricComputation: AUC over a trailing buffer of +# `lifetime_auc_window` samples/rank (legacy; per-rank buffer all-gathered). +# Override per-run via $TRAIN_LIFETIME_AUC_MODE / $EVAL_LIFETIME_AUC_MODE. +MetricsLogger.train_lifetime_auc_mode = @tlam/env_str() +tlam/env_str.key = "TRAIN_LIFETIME_AUC_MODE" +tlam/env_str.default = "binned" +MetricsLogger.eval_lifetime_auc_mode = @elam/env_str() +elam/env_str.key = "EVAL_LIFETIME_AUC_MODE" +elam/env_str.default = "binned" +# Score-histogram resolution for the "binned" backend. Higher = finer AUC +# resolution at O(bins) memory. Override via $CUMULATIVE_AUC_BINS. +MetricsLogger.cumulative_auc_bins = @cab/env_int() +cab/env_int.key = "CUMULATIVE_AUC_BINS" +cab/env_int.default = 100000 +# Trailing-buffer size (samples/rank) for the "capped" backend. Override via +# $LIFETIME_AUC_WINDOW. Ignored when the backend is "binned". +MetricsLogger.lifetime_auc_window = @law/env_int() +law/env_int.key = "LIFETIME_AUC_WINDOW" +law/env_int.default = 10000000 +# Checkpointing disabled by default — a full DMP checkpoint is ~100s of GB and +# the streaming loop always saves on the final window. save_dmp_checkpoint +# no-ops on the empty path. Set $CKPT_PATH to a directory to re-enable; the +# load path auto-resolves to the highest-numbered numeric subdir inside it. +save_dmp_checkpoint.path = @ckpt/env_path() +ckpt/env_path.key = "CKPT_PATH" +ckpt/env_path.default = "" +load_dmp_checkpoint.path = @ckpt/env_path() +# Retention: keep only the most-recent N numeric subdirs after each successful +# save (atomic rename + prune-older). Override via $KEEP_LAST_N. +save_dmp_checkpoint.keep_last_n = @kln/env_int() +kln/env_int.key = "KEEP_LAST_N" +kln/env_int.default = 1 +# In-window checkpoint cadence for streaming-train-eval. 0 (default) = end-of- +# window only; on crash mid-window the partial progress is lost. Set N>0 (e.g. +# via $IN_WINDOW_CKPT_FREQ) for batch-granularity exact-once-on-resume — paired +# with the auto-latest load above, the resumed run skips the K already-trained +# batches of the partial window and continues with bit-equal trajectory. +streaming_train_eval_loop.in_window_checkpoint_frequency = @iwcf/env_int() +iwcf/env_int.key = "IN_WINDOW_CKPT_FREQ" +iwcf/env_int.default = 0 +# Global-step checkpoint cadence: save whenever the monotonic train global_step +# crosses a multiple of N (a true "every 1000 steps" trigger that spans windows +# and survives resume). 0 (default) = off. Override via $CKPT_STEP_FREQ. +streaming_train_eval_loop.checkpoint_step_frequency = @csf/env_int() +csf/env_int.key = "CKPT_STEP_FREQ" +csf/env_int.default = 0 +# Wall-clock checkpoint cadence in seconds: save when >= this many seconds have +# elapsed since the last save. Rank 0 owns the clock and broadcasts the decision +# so all ranks save together. Default 3600 = hourly saves (matches the canonical +# fp16 run); set 0.0 to disable. Override via $CKPT_TIME_INTERVAL_S. +streaming_train_eval_loop.checkpoint_time_interval_s = @ctis/env_float() +ctis/env_float.key = "CKPT_TIME_INTERVAL_S" +ctis/env_float.default = 3600.0 +# Cap each train_ts window's batch count (mostly for the resume test driver). +# Unset / 0 = use the full window. +streaming_train_eval_loop.num_train_batches = @ntb/env_int() +ntb/env_int.key = "NUM_TRAIN_BATCHES" +ntb/env_int.default = 0 +# Cap each eval (full-holdout) window's batch count. Unset / <=0 = consume the +# full eval window (the genuine full-holdout NE/AUC; this is what the long run +# uses). Set >0 via $NUM_EVAL_BATCHES to subsample eval for fast validation. +streaming_train_eval_loop.num_eval_batches = @neb/env_int() +neb/env_int.key = "NUM_EVAL_BATCHES" +neb/env_int.default = 0 +# Test-only failure injection: when >=0 and metric_logger.global_step['train'] +# reaches this, the process exits with code 42 right after the in-window save +# fires. Used by the streaming resume test harness (train/tests/) to verify resume. +streaming_train_eval_loop.die_at_step = @das/env_int() +das/env_int.key = "DIE_AT_STEP" +das/env_int.default = -1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py new file mode 100644 index 000000000..8a716f87b --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py @@ -0,0 +1,413 @@ +# Copyright (c) 2026 Advanced Micro Devices, Inc. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +"""MLPerf Training compliance logging for the DLRMv3 streaming-train-eval path. + +Rank-0-gated wrapper around ``mlperf_logging.mllog`` so the streaming loop emits +the MLPerf event stream without every call site re-checking rank or the dep. +""" + +import logging +import os +from typing import Any, Dict, Optional + +import gin +import torch + +logger: logging.Logger = logging.getLogger(__name__) + +try: + from mlperf_logging import mllog + from mlperf_logging.mllog import constants as mllog_constants + + _MLLOG_AVAILABLE = True +except Exception as e: # pragma: no cover - import-time guard + mllog = None # type: ignore[assignment] + mllog_constants = None # type: ignore[assignment] + _MLLOG_AVAILABLE = False + logger.warning( + "mlperf_logging not importable (%s); MLPerf logging disabled. " + "Install via `pip install git+https://github.com/mlcommons/logging.git`.", + e, + ) + + +def _rank() -> int: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + return torch.distributed.get_rank() + return 0 + + +def _barrier() -> None: + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.barrier() + + +class MLPerfLogger: + """Rank-0-gated facade over ``mllog``. + + Event methods no-op on non-zero ranks and when mlperf_logging is absent. + ``sync=True`` barriers before emit so the timestamp reflects the slowest rank + (required for INIT_STOP/RUN_START/RUN_STOP). + """ + + def __init__( + self, + rank: Optional[int] = None, + log_path: Optional[str] = None, + default_stack_offset: int = 2, + benchmark_name: str = "hstu", + submitter_name: str = "AMD", + submission_platform: str = "MI355X", + fresh: bool = True, + ): + self.enabled: bool = _MLLOG_AVAILABLE + # Use the EXPLICIT caller rank: this is built before init_process_group, + # when dist.get_rank() would return 0 on every rank (all would log). + self.rank: int = rank if rank is not None else _rank() + self.benchmark_name: str = benchmark_name + self.submitter_name: str = submitter_name + self.submission_platform: str = submission_platform + self._logger = None + if not self.enabled: + return + # Only rank 0 emits, so only rank 0 needs the file handler. + if log_path and self.rank == 0: + log_dir = os.path.dirname(log_path) + if log_dir: # guard: os.makedirs("") raises for a bare filename + os.makedirs(log_dir, exist_ok=True) + # mllog's FileHandler APPENDS (mode "a"), which is what a resume needs + # so the single run's event stream accumulates across relaunches into + # one file. On a genuine cold start, truncate first so a re-used run + # dir / a previous crashed-cold-start's orphaned stream can't leave a + # second run_start in the file (the compliance checker requires + # EXACTLY_ONE). Resume (fresh=False) appends to continue the stream. + if fresh: + open(log_path, "w").close() + mllog.config(filename=log_path, default_stack_offset=default_stack_offset) + else: + mllog.config(default_stack_offset=default_stack_offset) + self._logger = mllog.get_mllogger() + + @property + def constants(self): # pyre-ignore[3] + return mllog_constants + + def event( + self, + key: str, + value: Any = None, + metadata: Optional[Dict[str, Any]] = None, + sync: bool = False, + ) -> None: + if sync: + _barrier() + if self.enabled and self.rank == 0: + self._logger.event(key=key, value=value, metadata=metadata or {}) + + def start( + self, + key: str, + value: Any = None, + metadata: Optional[Dict[str, Any]] = None, + sync: bool = False, + ) -> None: + if sync: + _barrier() + if self.enabled and self.rank == 0: + self._logger.start(key=key, value=value, metadata=metadata or {}) + + def end( + self, + key: str, + value: Any = None, + metadata: Optional[Dict[str, Any]] = None, + sync: bool = False, + ) -> None: + if sync: + _barrier() + if self.enabled and self.rank == 0: + self._logger.end(key=key, value=value, metadata=metadata or {}) + + def submission_info(self) -> None: + """Emit the five SUBMISSION_* events required for a valid submission.""" + if not (self.enabled and self.rank == 0): + return + c = mllog_constants + self.event(key=c.SUBMISSION_BENCHMARK, value=self.benchmark_name) + self.event(key=c.SUBMISSION_ORG, value=self.submitter_name) + self.event(key=c.SUBMISSION_DIVISION, value=c.CLOSED) + self.event(key=c.SUBMISSION_STATUS, value=c.ONPREM) + self.event(key=c.SUBMISSION_PLATFORM, value=self.submission_platform) + + def log_run_start( + self, + global_batch_size: int, + seed: int, + gradient_accumulation_steps: int = 1, + ) -> None: + """Emit submission info + core hyperparameters, then INIT_STOP + RUN_START. + + Optimizer names/LRs are read from gin (dense Adam + sparse RowWiseAdagrad), + resolving env-macro refs to concrete values. Call once on a genuine cold + start, after the model is built. INIT_STOP/RUN_START barrier so the + timestamp reflects the slowest rank, so ALL ranks must call this together + (non-rank-0 / disabled calls no-op the emit but still hit the barrier). + """ + c = self.constants + self.submission_info() + self.event(key=c.GLOBAL_BATCH_SIZE, value=int(global_batch_size)) + self.event( + key=c.GRADIENT_ACCUMULATION_STEPS, value=int(gradient_accumulation_steps) + ) + self.event(key=c.SEED, value=int(seed)) + self.event( + key=c.OPT_NAME, + value=_gin_param("dense_optimizer_factory_and_class.optimizer_name", "Adam"), + ) + self.event( + key=c.OPT_BASE_LR, + value=_gin_param("dense_optimizer_factory_and_class.learning_rate", None), + ) + self.event( + key="opt_sparse_name", + value=_gin_param( + "sparse_optimizer_factory_and_class.optimizer_name", "RowWiseAdagrad" + ), + ) + self.event( + key="opt_sparse_base_learning_rate", + value=_gin_param( + "sparse_optimizer_factory_and_class.learning_rate", None + ), + ) + self.end(key=c.INIT_STOP, sync=True) + self.start(key=c.RUN_START, sync=True) + + +def _gin_param(name: str, default: Any) -> Any: + """Read a gin-bound parameter, resolving env-macro refs to concrete values. + + Returns ``default`` if the parameter is unbound or a macro ref cannot be + resolved (so env-overridden LRs log as numbers, not unencodable objects). + """ + try: + value = gin.query_parameter(name) + except (ValueError, KeyError): + return default + if hasattr(value, "scoped_configurable_fn"): + try: + return value.scoped_configurable_fn() + except Exception: + return default + return value + + +class MLPerfRunTracker: + """Centralized MLPerf run-boundary state machine for the streaming loop. + + Owns the block/eval/run markers, the SAMPLES_COUNT/EPOCH_NUM progress + metadata, and the convergence decision (per-window AUC vs the configured + ``auc_threshold``). Every method no-ops when ``logger`` is None, so the + streaming loop can call them unconditionally. The convergence metric is + fixed to per-window AUC (higher-is-better). + """ + + # MetricsLogger.compute key short name for per-window AUC. + _EVAL_METRIC_SHORT = "window_auc" + + def __init__( + self, + logger: Optional[MLPerfLogger], + metric_logger: Any, + total_train_samples: int, + rank: int, + device: Any, + ): + self.logger = logger + self.metric_logger = metric_logger + self.total_train_samples = int(total_train_samples) + self.rank = int(rank) + self.device = device + self.run_stopped: bool = False + # Idempotency flag so the boundary helpers and the outer loop can both + # call start/stop without risking a double BLOCK_START/STOP. + self._block_open: bool = False + + @property + def enabled(self) -> bool: + return self.logger is not None + + def _progress(self) -> Dict[str, Any]: + c = self.logger.constants + samples = self.metric_logger.cumulative_train_samples + epoch = ( + samples / self.total_train_samples if self.total_train_samples > 0 else 0.0 + ) + return {c.SAMPLES_COUNT: samples, c.EPOCH_NUM: epoch} + + def log_dataset_sizes(self, eval_samples: Optional[int] = None) -> None: + if not self.enabled: + return + c = self.logger.constants + self.logger.event(key=c.TRAIN_SAMPLES, value=self.total_train_samples) + if eval_samples is not None: + self.logger.event(key=c.EVAL_SAMPLES, value=int(eval_samples)) + + def block_start(self) -> None: + if self.enabled and not self._block_open: + self.logger.start( + key=self.logger.constants.BLOCK_START, metadata=self._progress() + ) + self._block_open = True + + def block_stop(self) -> None: + if self.enabled and self._block_open: + self.logger.end( + key=self.logger.constants.BLOCK_STOP, metadata=self._progress() + ) + self._block_open = False + + def eval_start(self) -> None: + if self.enabled: + self.logger.start( + key=self.logger.constants.EVAL_START, metadata=self._progress() + ) + + def _target_metric(self, metrics: Dict[str, float]) -> Optional[float]: + # Key format `metric/{prefix}_{name}/{task}` (see MetricsLogger.compute); + # match the per-window AUC short name. + for key, val in metrics.items(): + short = key.split("/")[-2] if "/" in key else key + if short == self._EVAL_METRIC_SHORT: + return float(val) + return None + + def _meets_target(self, value: Optional[float]) -> bool: + thr = self.metric_logger.auc_threshold + if value is None or thr is None: + return False + return value >= thr + + def run_stop(self, status: object) -> None: + # Emit RUN_STOP exactly once, with an all-rank barrier so the timestamp + # reflects the slowest rank (MLPerf requirement). + if not self.enabled or self.run_stopped: + return + c = self.logger.constants + self.logger.end( + key=c.RUN_STOP, + metadata={c.STATUS: status, **self._progress()}, + sync=True, + ) + self.run_stopped = True + + def eval_stop(self, eval_metrics: Dict[str, float]) -> bool: + # Emit EVAL_ACCURACY + EVAL_STOP, early SUCCESS RUN_STOP on target. + # Rank 0 decides + broadcasts the stop bool so all ranks break in lockstep + # (a per-rank test could diverge and hang the next all-to-all). + if not self.enabled: + return False + c = self.logger.constants + eval_value = self._target_metric(eval_metrics) + if eval_value is not None: + self.logger.event( + key=c.EVAL_ACCURACY, value=eval_value, metadata=self._progress() + ) + self.logger.end(key=c.EVAL_STOP, metadata=self._progress()) + decision = torch.zeros(1, device=self.device) + if self.rank == 0 and not self.run_stopped and self._meets_target(eval_value): + decision[0] = 1.0 + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.broadcast(decision, src=0) + should_stop = bool(decision.item() > 0.5) + if should_stop: + # All ranks agree -> all reach the RUN_STOP barrier together. + self.run_stop(c.SUCCESS) + return should_stop + + def finalize(self, final_metrics: Dict[str, float]) -> None: + # End-of-run RUN_STOP when the target was never crossed: SUCCESS iff the + # final eval metric meets the target, else ABORTED. + if not self.enabled or self.run_stopped: + return + c = self.logger.constants + success = self._meets_target(self._target_metric(final_metrics)) + self.run_stop(c.SUCCESS if success else c.ABORTED) + + +def mlperf_checkpoint_present(ckpt_path: str) -> bool: + """True iff ``ckpt_path`` resolves to an existing checkpoint (i.e. a resume). + + A dependency-light mirror of ``checkpoint._resolve_latest_subdir`` so + ``train_ranker`` can decide cold-start vs resume BEFORE the heavy checkpoint + import + ``setup()``. This gates the one-time INIT_START/RUN_START markers: + emit them on a genuine cold start only, and never re-emit on a resume + relaunch (the MLPerf run spans the resume). Matches the loader's resolution: + empty path or a base dir with no numeric subdirs => cold start. + """ + if not ckpt_path: + return False + base = ckpt_path.rstrip("/") + # A leaf save (numeric basename) is a resume iff that dir actually exists. + if os.path.basename(base).isdigit(): + return os.path.isdir(base) + if not os.path.isdir(base): + return False + for name in os.listdir(base): + if name.isdigit() and os.path.isdir(os.path.join(base, name)): + return True + return False + + +@gin.configurable +def get_mlperf_logger( + rank: int = 0, + log_path: str = "", + benchmark_name: str = "hstu", + submitter_name: str = "AMD", + submission_platform: str = "MI355X", + fresh: bool = True, +) -> Optional[MLPerfLogger]: + """Build a configured :class:`MLPerfLogger`, or ``None`` if unavailable. + + Path defaults to ``$MLPERF_LOG_PATH``. Returns ``None`` (not a disabled + logger) so callers' ``is not None`` guards cleanly skip logging. + + Disable knob: set ``$MLPERF_LOGGING=0`` (or false/no/off) to turn the whole + MLPerf event stream off — returns ``None`` on EVERY rank, so the train loop's + ``is not None`` guards skip emission AND the cross-rank train-loss all-reduce + in lockstep. Default (unset / "1") = enabled, preserving prior behavior. + """ + if not _MLLOG_AVAILABLE: + return None + if os.environ.get("MLPERF_LOGGING", "1").strip().lower() in ( + "0", "false", "no", "off", + ): + logger.info("MLPerf logging disabled via $MLPERF_LOGGING=0") + return None + resolved_path = os.environ.get("MLPERF_LOG_PATH", log_path) + # SUBMISSION_PLATFORM defaults to "MI355X"; override per-submitter via env. + resolved_platform = os.environ.get( + "MLPERF_SUBMISSION_PLATFORM", submission_platform + ) + return MLPerfLogger( + rank=rank, + log_path=resolved_path, + benchmark_name=benchmark_name, + submitter_name=submitter_name, + submission_platform=resolved_platform, + fresh=fresh, + ) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/checkpoint_cadence_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/checkpoint_cadence_test.py new file mode 100644 index 000000000..4a483c262 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/checkpoint_cadence_test.py @@ -0,0 +1,131 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +"""Unit tests for `select_in_window_checkpoint_reason` — the pure decision that +drives the streaming loop's three fine-grained checkpoint cadences: + + * `in_window_checkpoint_frequency` — per-window-local batch count + * `checkpoint_step_frequency` — monotonic global step ("every 1000 steps") + * `checkpoint_time_interval_s` — wall-clock ("hourly") + +These run without a GPU / distributed init: the loop broadcasts a single +`elapsed_since_last_save` from rank 0 and then calls this pure function, so +exercising the function directly fully covers the trigger semantics. +""" +import unittest + +from generative_recommenders.dlrm_v3.train.utils import ( + select_in_window_checkpoint_reason, +) + + +def _reason( + *, + batch: int = 1, + step: int = 1, + elapsed: float = 0.0, + in_window: int = 0, + step_freq: int = 0, + time_s: float = 0.0, +) -> str | None: + return select_in_window_checkpoint_reason( + train_batch_idx=batch, + global_step=step, + elapsed_since_last_save=elapsed, + in_window_checkpoint_frequency=in_window, + checkpoint_step_frequency=step_freq, + checkpoint_time_interval_s=time_s, + ) + + +class CheckpointCadenceTest(unittest.TestCase): + def test_all_disabled_never_fires(self) -> None: + for batch in (1, 100, 1000): + for step in (1, 1000, 5000): + self.assertIsNone(_reason(batch=batch, step=step, elapsed=1e9)) + + def test_step_based_every_1000(self) -> None: + # Fires exactly on multiples of the step frequency. + self.assertEqual(_reason(step=1000, step_freq=1000), "global_step") + self.assertEqual(_reason(step=2000, step_freq=1000), "global_step") + # Does not fire just off a boundary. + self.assertIsNone(_reason(step=999, step_freq=1000)) + self.assertIsNone(_reason(step=1001, step_freq=1000)) + + def test_step_zero_does_not_trigger(self) -> None: + # global_step==0 must not trivially satisfy `0 % N == 0`. + self.assertIsNone(_reason(step=0, step_freq=1000)) + + def test_time_based_interval(self) -> None: + # At/over the interval -> fires; under -> no save. + self.assertEqual( + _reason(step=3, elapsed=3600.0, time_s=3600.0), "time_interval" + ) + self.assertEqual( + _reason(step=3, elapsed=4000.0, time_s=3600.0), "time_interval" + ) + self.assertIsNone(_reason(step=3, elapsed=3599.9, time_s=3600.0)) + + def test_in_window_batch_cadence(self) -> None: + self.assertEqual(_reason(batch=5, in_window=5), "in_window_batch") + self.assertEqual(_reason(batch=10, in_window=5), "in_window_batch") + self.assertIsNone(_reason(batch=4, in_window=5)) + + def test_precedence_in_window_over_step_over_time(self) -> None: + # All three would fire this batch; precedence picks in_window first. + self.assertEqual( + _reason( + batch=5, + step=1000, + elapsed=9999.0, + in_window=5, + step_freq=1000, + time_s=3600.0, + ), + "in_window_batch", + ) + # in_window not due this batch -> step wins over time. + self.assertEqual( + _reason( + batch=4, + step=1000, + elapsed=9999.0, + in_window=5, + step_freq=1000, + time_s=3600.0, + ), + "global_step", + ) + # Neither batch nor step due -> time wins. + self.assertEqual( + _reason( + batch=4, + step=999, + elapsed=9999.0, + in_window=5, + step_freq=1000, + time_s=3600.0, + ), + "time_interval", + ) + + def test_step_and_time_combined_independent(self) -> None: + # Step frequency enabled, time disabled: only step boundaries fire. + self.assertEqual(_reason(step=1000, step_freq=1000, time_s=0.0), "global_step") + self.assertIsNone(_reason(step=1000, elapsed=1e9, step_freq=0, time_s=0.0)) + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py new file mode 100644 index 000000000..e3f78886c --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py @@ -0,0 +1,287 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""End-to-end failure-injection test for streaming resume. + +Two scenarios, driven by the sibling `streaming_resume_test.sh` (see its header +for the full, platform-general launch wiring — NVIDIA B200 and AMD MI350/MI355). +This module is the shared log parser + a CLI the driver shells out to. + +SCENARIO `midwindow` — exact-once mid-window resume. Validates the four +single-window resume features end-to-end on the yambda-5b stack: + 1. Mid-window save (in_window_checkpoint_frequency) + 2. Within-window exact-once skip (StreamingWindowSampler.set_window skip) + 3. Auto-detect-latest checkpoint subdir + 4. keep_last_n retention (default 1) + +SCENARIO `multiwindow` — distributed-sync regression guard for the two fixes the +mid-window test cannot reach (it runs ONE window with per-window eval off): + A. total_train_anchors() computed once on rank 0 + broadcast (not world_size×). + B. window-boundary dist.barrier() before the first forward of each window. +Both only matter across >=2 windows with the data-fraction eval cadence +(EVAL_EVERY_DATA_PCT>0) active, and the deadlock they fix originally struck at a +window boundary mid-run — so the scenario trains multiple windows AND resumes +across a completed-window boundary. The signals are extracted by `summarize` +(see `summarize_run`) and asserted in the shell driver. + +Test flow (driven by the sibling `streaming_resume_test.sh`): + Phase 1 (baseline): Run streaming-train-eval for N=2 train_ts × K batches/window + with die_at_step=-1. Capture per-batch window_ne / window_auc into traj_baseline.json. + Phase 2 (interrupt): Same config but die_at_step=M (M mid-window-2). Expect + process to exit(42) after the in-window checkpoint at step M lands. + Phase 3 (resume): Re-launch with same CKPT_PATH (auto-latest picks the + in-window save). Continue to the same total step count. Capture + traj_resumed.json (which only contains the post-resume steps). + + Correctness is proven by the FUNCTIONAL INVARIANTS checked in the shell + driver (resumed at exactly batch_idx_in_window, per-rank RNG restored, atomic + save + keep_last_n), NOT by bit-equal trajectory matching. The training stack + is nondeterministic across runs (non-deterministic atomic scatter-add in the + embedding/attention backward on ROCm): two independent *cold* runs already + drift ~7e-4 in window_ne over 20 steps, and early-training chaos amplifies + it, so resume-vs-baseline can differ by a few percent even when resume is + perfect. The trajectory comparison here is therefore a LOOSE closeness bound + (default atol below) that only flags gross divergence — wrong data slice or + unrestored model state — while tolerating nondeterministic drift. + +This module also provides a CLI entry point used by the shell driver to (a) +parse a train.log into a step-keyed dict of metrics, and (b) compare two such +dicts and fail loudly on mismatch. +""" + +import argparse +import json +import re +import sys +from typing import Dict, List, Optional, Tuple + +# Per-step metrics from MetricsLogger.compute_and_log are emitted like: +# "train - Step 51 metrics: {'metric/lifetime_ne/listen_plus': tensor(1.0954, ...) +# 'metric/window_ne/listen_plus': tensor(0.9940, ...), +# 'metric/window_accuracy/listen_plus': tensor(0.6231, ...) ..." +_STEP_RE = re.compile(r"train - Step (\d+) metrics:") +_WNE_RE = re.compile(r"window_ne/listen_plus.*?tensor\(([0-9.]+)") +_WAUC_RE = re.compile(r"window_auc/listen_plus.*?tensor\(([0-9.]+)") +_WACC_RE = re.compile(r"window_accuracy/listen_plus.*?tensor\(([0-9.]+)") + + +# --- multi-window / data-pct-eval regression signals ------------------------- +# These cover the two distributed-sync fixes that the single-window mid-window +# test above does NOT exercise (it runs one window with per-window eval off): +# +# (A) total_train_anchors() rank-0 broadcast. The data-fraction eval cadence +# needs total_train_anchors — a multi-minute, single-threaded O(N) gather +# + uid-hash over the mmap'd anchor array. Run on EVERY rank it both wastes +# 8x CPU and desyncs the NCCL stream (a fast rank races into the first +# embedding all-to-all while slow ranks still hash) → deadlock. The fix +# computes it ONCE on rank 0 and broadcasts the scalar. yambda logs exactly +# one `total_train_anchors(start_ts=…)` line per call, so the regression +# guard is: that line appears EXACTLY ONCE per launch (was world_size×). +# +# (B) window-boundary barrier. Per-window data prep (`window_indices`, an O(N) +# mask over the ~18GB mmap) finishes at very different times across ranks; +# without a sync before the first forward the collective stream desyncs and +# the job hangs at the boundary. The fix adds a dist.barrier() at each +# window boundary. It is silent on the healthy path, so the trainer emits a +# `[window-barrier] … rendezvous complete` line (rank 0) per crossed window +# ONLY under WINDOW_BARRIER_DEBUG=1 — the guard counts those == #windows. +_TTA_RE = re.compile(r"total_train_anchors\(start_ts=(\d+),\s*num_ts=(\d+)\):") +_BARRIER_RE = re.compile(r"\[window-barrier\] train_ts=(\d+) rendezvous complete") +_DATA_PCT_SETUP_RE = re.compile( + r"\[data-pct-eval\] eval_every_data_pct=.*?eval_interval_steps=(\d+)" +) +_DATA_PCT_TRIGGER_RE = re.compile(r"\[data-pct-eval\] trigger eval train_ts=(\d+)") +_RESUME_COMPLETED_RE = re.compile(r"Resuming from completed train_ts=(\d+)") +_RESUME_MIDWINDOW_RE = re.compile( + r"Resuming mid-window at train_ts=(\d+) batch_idx_in_window=(\d+)" +) +# Test driver appends this sentinel after the trainer returns (clean OR crash); +# code 0 == the run finished all requested windows + final eval without hanging. +_PHASE_EXIT_RE = re.compile(r"PHASE_EXIT=(-?\d+)") + + +def summarize_run(log_path: str) -> Dict[str, object]: + """Extract the multi-window / data-pct-eval regression signals from a run log. + + Returns a JSON-able dict the shell driver asserts on. All counts are over the + WHOLE log (one launch's worth — the driver uses a fresh per-phase log).""" + tta_calls: List[Tuple[int, int]] = [] + barrier_windows: List[int] = [] + data_pct_eval_setup: bool = False + data_pct_eval_interval: Optional[int] = None + data_pct_eval_triggers: List[int] = [] + resume_completed_ts: Optional[int] = None + resume_midwindow: Optional[Tuple[int, int]] = None + phase_exit: Optional[int] = None + with open(log_path, "r", errors="replace") as f: + for line in f: + m = _TTA_RE.search(line) + if m: + tta_calls.append((int(m.group(1)), int(m.group(2)))) + m = _BARRIER_RE.search(line) + if m: + barrier_windows.append(int(m.group(1))) + m = _DATA_PCT_SETUP_RE.search(line) + if m: + data_pct_eval_setup = True + data_pct_eval_interval = int(m.group(1)) + m = _DATA_PCT_TRIGGER_RE.search(line) + if m: + data_pct_eval_triggers.append(int(m.group(1))) + m = _RESUME_COMPLETED_RE.search(line) + if m: + resume_completed_ts = int(m.group(1)) + m = _RESUME_MIDWINDOW_RE.search(line) + if m: + resume_midwindow = (int(m.group(1)), int(m.group(2))) + m = _PHASE_EXIT_RE.search(line) + if m: + phase_exit = int(m.group(1)) + return { + # (A) rank-0 broadcast: must be exactly 1 (was world_size× before the fix) + "total_train_anchors_calls": len(tta_calls), + "total_train_anchors_args": tta_calls, + # (B) barrier executed once per crossed window (rank 0, debug-gated) + "window_barrier_count": len(barrier_windows), + "windows_trained": sorted(set(barrier_windows)), + # data-fraction eval cadence active + actually fired + "data_pct_eval_setup": data_pct_eval_setup, + "data_pct_eval_interval_steps": data_pct_eval_interval, + "data_pct_eval_trigger_count": len(data_pct_eval_triggers), + # resume classification + "resume_completed_ts": resume_completed_ts, + "resume_midwindow": resume_midwindow, + # terminal status (None => still running / killed without sentinel) + "phase_exit": phase_exit, + } + + +def parse_trajectory(log_path: str) -> Dict[int, Dict[str, float]]: + """Extract a {step: {window_ne, window_auc, window_accuracy}} dict from a + train.log. The grep is loose on the metric line itself — we accept the + very long truncated form MetricsLogger prints.""" + out: Dict[int, Dict[str, float]] = {} + with open(log_path, "r", errors="replace") as f: + for line in f: + m = _STEP_RE.search(line) + if not m: + continue + step = int(m.group(1)) + wne = _WNE_RE.search(line) + wauc = _WAUC_RE.search(line) + wacc = _WACC_RE.search(line) + if not (wne and wauc and wacc): + continue + # Only keep ONE entry per step — log can have duplicate per-rank + # prints; first one wins (they're identical). + if step in out: + continue + out[step] = { + "window_ne": float(wne.group(1)), + "window_auc": float(wauc.group(1)), + "window_accuracy": float(wacc.group(1)), + } + return out + + +def compare_trajectories( + baseline: Dict[int, Dict[str, float]], + resumed: Dict[int, Dict[str, float]], + min_resume_step: int, + atol: float = 0.15, +) -> Tuple[bool, str]: + """Compare baseline vs resumed trajectories for steps >= min_resume_step. + + This is a LOOSE closeness bound, not a bit-equality check — see the module + docstring. `atol` defaults to a value that tolerates the nondeterministic + cross-run drift of this stack while still catching gross resume bugs. + Returns (ok, message). `ok=False` on any divergence outside `atol`.""" + steps = sorted(s for s in resumed if s >= min_resume_step) + if not steps: + return False, f"No resumed steps >= {min_resume_step}" + mismatches = [] + for s in steps: + if s not in baseline: + mismatches.append(f"step {s}: missing from baseline") + continue + b = baseline[s] + r = resumed[s] + for key in ("window_ne", "window_auc", "window_accuracy"): + if abs(b[key] - r[key]) > atol: + mismatches.append( + f"step {s} {key}: baseline={b[key]:.6f} " + f"resumed={r[key]:.6f} diff={b[key]-r[key]:+.6f}" + ) + if mismatches: + return False, ( + f"{len(mismatches)} mismatches across {len(steps)} resumed steps " + f"(atol={atol}):\n " + "\n ".join(mismatches[:10]) + ) + return True, ( + f"{len(steps)} resumed steps match baseline within atol={atol} " + f"(range: step {steps[0]}..{steps[-1]})" + ) + + +def main() -> int: + ap = argparse.ArgumentParser() + sub = ap.add_subparsers(dest="cmd", required=True) + + p_parse = sub.add_parser("parse", help="Parse a train.log → traj JSON") + p_parse.add_argument("log") + p_parse.add_argument("out") + + p_cmp = sub.add_parser("compare", help="Compare baseline vs resumed traj JSONs") + p_cmp.add_argument("baseline") + p_cmp.add_argument("resumed") + p_cmp.add_argument("--min-resume-step", type=int, required=True) + p_cmp.add_argument("--atol", type=float, default=0.15) + + p_sum = sub.add_parser( + "summarize", + help="Emit multi-window / data-pct-eval regression signals from a run log", + ) + p_sum.add_argument("log") + p_sum.add_argument("out", nargs="?", help="optional JSON output path") + + args = ap.parse_args() + if args.cmd == "parse": + traj = parse_trajectory(args.log) + with open(args.out, "w") as f: + json.dump(traj, f, indent=2) + print(f"Wrote {len(traj)} step entries to {args.out}", file=sys.stderr) + return 0 + if args.cmd == "compare": + with open(args.baseline) as f: + baseline = {int(k): v for k, v in json.load(f).items()} + with open(args.resumed) as f: + resumed = {int(k): v for k, v in json.load(f).items()} + ok, msg = compare_trajectories( + baseline, resumed, args.min_resume_step, atol=args.atol + ) + print(msg) + return 0 if ok else 1 + if args.cmd == "summarize": + summary = summarize_run(args.log) + out = json.dumps(summary, indent=2) + if args.out: + with open(args.out, "w") as f: + f.write(out) + print(out) + return 0 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh new file mode 100755 index 000000000..e093fb31a --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh @@ -0,0 +1,566 @@ +#!/bin/bash +# End-to-end failure-injection + resume test for streaming-train-eval. +# +# ============================================================================ +# WHY TWO TESTS (the intuition, in plain language) +# ============================================================================ +# Training runs over consecutive time WINDOWS (window 0, then 1, then 2, ...). +# All N GPUs must march from one window to the next IN LOCKSTEP: they constantly +# do "everybody-talk-at-once" group ops (NCCL collectives — sharing embeddings +# across GPUs), and every GPU must enter each group-op at the same time. If one +# GPU is late, the rest wait for it forever and the whole job FREEZES (deadlock). +# +# The two scenarios check DIFFERENT kinds of failure — not bigger vs smaller: +# +# midwindow = CORRECTNESS. "When I crash and resume, do I land on the exact +# right batch with the right RNG state and produce the same +# numbers?" (stays inside ONE window; never crosses a seam.) +# +# multiwindow = LIVENESS / NO-DEADLOCK. "Can all N GPUs hand off across a +# window SEAM together without one falling out of step and +# hanging the job?" (needs >=2 windows so a seam actually exists.) +# +# The dangerous spot is the SEAM between two windows: there, each GPU does solo +# prep work (load next window's data; count anchors for the eval cadence) and +# they DON'T all finish at the same speed. Two bugs lived exactly there, and BOTH +# are invisible to a single-window test: +# (A) every GPU separately ran a slow O(N) "count all the data" pass -> they +# finished at different times -> fast GPU barged into the next group-op +# while others were still counting -> freeze. +# FIX: only rank 0 counts, then broadcasts the number to everyone else. +# (B) no rendezvous at the seam -> uneven data-prep -> same desync -> freeze. +# FIX: a dist.barrier() at every window boundary (all GPUs wait, then cross +# together). WINDOW_BARRIER_DEBUG=1 makes rank 0 log one line per seam. +# +# TIMELINE — without the fixes (each GPU on its own clock at the seam): +# win0 train | solo prep (varies) | next group-op +# GPU0 ########|=====| >> waiting.......... +# GPU1 ########|========| >> waiting....... +# GPU2 ########|===========| >> waiting.... +# GPU3 ########|==============| >> never lines up -> HANG +# +# TIMELINE — with the fixes (rank 0 counts + a barrier gate at the seam): +# win0 train | [== BARRIER: all wait ==] | win1 train +# GPU0 ########| count | wait # |######## +# GPU1 ########| | wait # |######## +# GPU2 ########| | wait # |######## +# GPU3 ########| | wait # |######## +# rank0 shares count ^ all cross together ^ -> OK +# +# Why midwindow can NOT catch (A)/(B): it runs a SINGLE window with per-window +# eval off (NUM_TRAIN_TS=1, EVAL_EVERY_N_WINDOWS=0, split=1.0), so it never +# reaches a seam and never turns on the data-fraction-eval/anchor-count path. +# A broken barrier or broken broadcast passes midwindow silently. +# +# Why the multiwindow RESUME phase (P3 below) is the meanest case: restarting +# from a checkpoint loads the saved window and then IMMEDIATELY steps across a +# seam into the next window — landing right on the spot that used to freeze, AND +# re-running all that slow setup on the resume path. If (A)/(B) regressed, P3 +# hangs and the test fails by timing out. +# +# | midwindow | multiwindow +# --------------+----------------------+----------------------------- +# proves | resume to RIGHT spot | cross seam WITHOUT freezing +# windows | 1 (no seam) | >=2 (crosses >=1 seam) +# data-pct eval | off | on (exercises the anchor count) +# catches | wrong batch/RNG/ckpt | missing barrier/broadcast -> HANG +# failure mode | wrong NUMBERS | job FREEZES forever +# They are complementary: you need BOTH. +# ============================================================================ +# +# PLATFORM-GENERAL: runs on both NVIDIA B200 and AMD MI350/MI355 (ROCm/meta64). +# The only hardware-specific bits are picked by --platform (auto-detected from the +# running container if omitted): the container name, the dataset path, and the +# checkpoint root. Everything else — the worker entrypoint (scripts/launch_slurm.sh, +# which is the shared launcher both clusters' supervisors use), the env-driven gin +# knobs, and all assertions — is identical across platforms. +# +# Two scenarios (select with --scenario; default runs both): +# +# midwindow — exact-once MID-WINDOW resume (single window). +# P1 baseline: uninterrupted 1 train_ts × K batches. +# P2 interrupted: same + die_at_step=M → exits AFTER the in-window ckpt at M. +# P3 resume: relaunch w/ same CKPT_PATH → auto-latest picks the in-window +# save, skips the M already-trained batches, finishes. +# Gates: re-entered at batch_idx_in_window=M, per-rank RNG restored, first +# resumed step == M+1, atomic save + keep_last_n, trajectory within --atol. +# +# multiwindow — distributed-sync REGRESSION guard for the two fixes the +# mid-window test cannot reach (it runs ONE window with per-window eval off): +# (A) total_train_anchors() computed ONCE on rank 0 + broadcast (the +# data-fraction eval cadence needs it; running the multi-minute O(N) +# mmap gather + uid-hash on every rank desynced NCCL → boundary hang). +# (B) a dist.barrier() at every window boundary before the first forward +# (per-rank data-prep skew otherwise desyncs the collective stream). +# Both only bite across >=2 windows with EVAL_EVERY_DATA_PCT>0, and the +# deadlock struck at a boundary mid-run, so: +# P1 mw_baseline: cold run over MW_TS windows w/ data-pct eval. Asserts +# total_train_anchors logged EXACTLY ONCE (computed at setup + broadcast +# from rank 0), the barrier fired on EVERY window, the data-pct cadence +# was set up, and the run COMPLETED (no boundary hang). +# P2 mw_seed: 1 window → clean end-of-window (WINDOW_COMPLETE) ckpt. +# P3 mw_resume: relaunch over MW_TS windows w/ same CKPT_PATH → resumes +# past the completed window and CROSSES the boundary into the next +# windows. Asserts "Resuming from completed", barrier fired on each +# remaining window, anchors broadcast once, and the run COMPLETED — +# i.e. the exact boundary-crossing-on-resume case that used to hang. +# +# Driven entirely via env-driven gin knobs (yambda_5b.gin) through the SAME worker +# entrypoint both platforms' production supervisors use: `bash scripts/launch_slurm.sh` +# (worker phase, auto-detected inside the container). WINDOW_BARRIER_DEBUG=1 makes +# the otherwise-silent barrier emit one rank-0 line per crossed window. +# +# CHECKPOINT/DATASET PLACEMENT (the one real platform difference): +# * B200: virtiofs/NFS WEDGES under the trainer's concurrent mmap LOAD, so the +# checkpoint root AND the mmap'd dataset cache MUST be node-local (defaults +# /tmp/...). The dataset must already be staged node-local at --data-path +# (the e2e supervisor's stage_data_in does this); the test fails fast if not. +# * MI350/MI355 (meta64): NFS mmap is fine, so the checkpoint root + dataset +# read directly from shared NFS (defaults /apps/chcai/...), as the original +# test did. No staging needed. +# Logs always use read()/write() only, so they live on shared /apps/chcai and +# are grep-able from the head node on both platforms. +# +# Usage: +# bash generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh \ +# --jobid [--platform b200|mi350] [--scenario all] +# [--container ] [--data-path ] [--ckpt-root ] [--start-ts 150] +# [--num-train-batches 200] [--die-at-step 100] # midwindow knobs +# [--mw-num-train-ts 3] [--mw-num-train-batches 20] # multiwindow knobs +# [--mw-eval-pct 0.34] [--phase-timeout S] [--mw-run-timeout S] [--keep] +# --platform is auto-detected from the running container when omitted. Any of +# --container/--data-path/--ckpt-root override the platform default. +# Per-phase wait budgets default per-platform (B200 node-local NVMe: 1800/3600s; +# MI350/MI355 shared-NFS full-model ckpts ~9 min each: 5400/5400s) and can be +# overridden with --phase-timeout (midwindow) / --mw-run-timeout (multiwindow). + +set -uo pipefail + +JOBID="" +# Repo root is derived from THIS script's location +# (/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.sh — +# four levels up), so the test is not pinned to any one user's home. Override with +# --repo if the repo is mounted at a different path inside the container. +_SELF_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO=$(cd "$_SELF_DIR/../../../.." && pwd) +DATASET_SUBDIR=processed_5b/hstu_cache_L4086 +SCENARIO=all # midwindow | multiwindow | all +START_TS=150 +KEEP=0 +LOG_DIR=/apps/chcai/streaming_resume_test # shared NFS (read()/write() only) +# Platform + the three platform-specific paths. Empty sentinels here; filled by +# apply_platform_defaults() AFTER platform detection unless the user overrode +# them on the command line. (DATA_PATH uses a distinct sentinel because an +# explicit empty value is meaningful: "do not inject DLRM_DATA_PATH; let the gin +# default apply".) +PLATFORM="" # b200 | mi350 | mi355 ; auto if empty +CONTAINER="" # default: per-platform +DATA_PATH="__AUTO__" # default: per-platform +CKPT_ROOT="" # default: per-platform (node-local on B200) + +# --- midwindow knobs --- +NUM_TRAIN_BATCHES=200 +NUM_EVAL_BATCHES=5 # cap the per-phase FINAL eval (0 = full holdout, very slow) +DIE_AT_STEP=100 +IN_WINDOW_FREQ=50 +ATOL=0.15 # trajectory closeness bound (NOT bit-equality; see py module) +# Per-phase wait budget. Left empty here and filled per-platform below (a B200 +# ckpt save/load hits node-local NVMe and is fast; on meta64 each full-model DCP +# save/load lands on shared NFS and takes ~9 min, and the resume phase does a +# LOAD + several in-window saves + an end-of-window save, so it needs far longer). +# Override explicitly with --phase-timeout. +MW_TIMEOUT="" + +# --- multiwindow knobs --- +MW_TS=3 # windows to train (>=2 to cross a boundary) +MW_BATCHES=20 # train batches per window (small = fast) +MW_EVAL_BATCHES=5 # holdout eval batches per fired eval +MW_EVAL_PCT=0.34 # data-fraction eval cadence (>0 enables the anchors path) +MW_SPLIT=0.90 # train split (<1 => holdout exists => uid-hash anchor path) +MW_HOLDOUT_TS=200 # PINNED holdout window (must match across seed→resume) +# generous: init + planner + anchors gather can take min; on NFS add ckpt save/load. +# Empty => filled per-platform below. Override with --mw-run-timeout. +MW_RUN_TIMEOUT="" + +while [[ $# -gt 0 ]]; do + case $1 in + --jobid) JOBID="$2"; shift 2;; + --platform) PLATFORM="$2"; shift 2;; + --container) CONTAINER="$2"; shift 2;; + --repo) REPO="$2"; shift 2;; + --data-path) DATA_PATH="$2"; shift 2;; + --dataset-subdir) DATASET_SUBDIR="$2"; shift 2;; + --scenario) SCENARIO="$2"; shift 2;; + --start-ts) START_TS="$2"; shift 2;; + --ckpt-root) CKPT_ROOT="$2"; shift 2;; + --log-dir) LOG_DIR="$2"; shift 2;; + --num-train-batches) NUM_TRAIN_BATCHES="$2"; shift 2;; + --num-eval-batches) NUM_EVAL_BATCHES="$2"; shift 2;; + --die-at-step) DIE_AT_STEP="$2"; shift 2;; + --in-window-freq) IN_WINDOW_FREQ="$2"; shift 2;; + --atol) ATOL="$2"; shift 2;; + --phase-timeout) MW_TIMEOUT="$2"; shift 2;; + --mw-run-timeout) MW_RUN_TIMEOUT="$2"; shift 2;; + --mw-num-train-ts) MW_TS="$2"; shift 2;; + --mw-num-train-batches) MW_BATCHES="$2"; shift 2;; + --mw-num-eval-batches) MW_EVAL_BATCHES="$2"; shift 2;; + --mw-eval-pct) MW_EVAL_PCT="$2"; shift 2;; + --mw-split) MW_SPLIT="$2"; shift 2;; + --mw-holdout-ts) MW_HOLDOUT_TS="$2"; shift 2;; + --keep) KEEP=1; shift;; + *) echo "Unknown arg: $1"; exit 1;; + esac +done +[[ -z "$JOBID" ]] && { echo "Error: --jobid required"; exit 1; } +case "$SCENARIO" in midwindow|multiwindow|all) ;; *) echo "Error: --scenario must be midwindow|multiwindow|all"; exit 1;; esac +(( MW_TS < 2 )) && { echo "Error: --mw-num-train-ts must be >=2 to cross a boundary"; exit 1; } +[[ -n "$PLATFORM" ]] && case "$PLATFORM" in b200|mi350|mi355) ;; *) echo "Error: --platform must be b200|mi350|mi355"; exit 1;; esac + +# --- resolve platform + its three hardware-specific paths -------------------- +# Precedence: explicit --platform > inferred from explicit --container > probe +# the allocation's docker for a known training container > default b200. +if [[ -z "$PLATFORM" ]]; then + if [[ "$CONTAINER" == "yambda_b200" ]]; then PLATFORM=b200 + elif [[ "$CONTAINER" == "yambda_primus" ]]; then PLATFORM=mi350 + else + _names=$(srun --jobid="$JOBID" --overlap docker ps -a --format '{{.Names}}' 2>/dev/null) + if grep -qx yambda_b200 <<<"$_names"; then PLATFORM=b200 + elif grep -qx yambda_primus <<<"$_names"; then PLATFORM=mi350 + else + # No known training container yet (e.g. container not provisioned). + # Fall back to probing the allocation's GPU vendor on the host so we + # do NOT silently assume a platform. + _vendor=$(srun --jobid="$JOBID" --overlap bash -lc \ + 'if command -v rocm-smi >/dev/null 2>&1; then echo amd; \ + elif command -v nvidia-smi >/dev/null 2>&1; then echo nvidia; \ + else echo unknown; fi' 2>/dev/null | head -1) + case "$_vendor" in + amd) PLATFORM=mi350; echo "[$(date)] no known container — detected AMD GPU host (rocm-smi) → mi350";; + nvidia) PLATFORM=b200; echo "[$(date)] no known container — detected NVIDIA GPU host (nvidia-smi) → b200";; + *) echo "Error: could not auto-detect platform on job $JOBID (no yambda_b200/yambda_primus container and no rocm-smi/nvidia-smi). Pass --platform b200|mi350|mi355."; exit 1;; + esac + fi + fi + echo "[$(date)] auto-detected platform: $PLATFORM" +fi +case "$PLATFORM" in + b200) + : "${CONTAINER:=yambda_b200}" + # B200: mmap (ckpt LOAD + dataset cache) must NOT touch virtiofs/NFS. + [[ "$DATA_PATH" == "__AUTO__" ]] && DATA_PATH=/tmp/yambda_data + : "${CKPT_ROOT:=/tmp/yambda_resume_test/ckpts}" + # Node-local NVMe: full-model save/load is fast. + : "${MW_TIMEOUT:=1800}" + : "${MW_RUN_TIMEOUT:=3600}" + ;; + mi350|mi355) + : "${CONTAINER:=yambda_primus}" + # meta64: NFS mmap is fine — read dataset + write ckpt directly on NFS + # (matches the original MI350 test). /apps/chcai/dlrm_data is the gin default. + [[ "$DATA_PATH" == "__AUTO__" ]] && DATA_PATH=/apps/chcai/dlrm_data + : "${CKPT_ROOT:=/apps/chcai/ckpts_resume_test}" + # Shared NFS: each full-model DCP save/load is ~9 min. The midwindow resume + # phase chains a LOAD + multiple in-window saves + an end-of-window save + # (>2000s observed), so the B200 budgets are far too tight — abandoning a + # still-running trainer leaks GPU VRAM and OOMs the next phase. Be generous. + : "${MW_TIMEOUT:=5400}" + : "${MW_RUN_TIMEOUT:=5400}" + ;; +esac +echo "[$(date)] platform=$PLATFORM container=$CONTAINER data_path=${DATA_PATH:-} ckpt_root=$CKPT_ROOT phase_timeout=${MW_TIMEOUT}s mw_run_timeout=${MW_RUN_TIMEOUT}s" + +mkdir -p "$LOG_DIR" +PYHELPER="$REPO/generative_recommenders/dlrm_v3/train/tests/streaming_resume_test.py" + +# --- container helpers (inspect CKPT/dataset via docker exec — works whether the +# path is node-local on B200 or shared NFS on MI350) --- +sx() { srun --jobid="$JOBID" --overlap docker exec "$CONTAINER" bash -lc "$1" 2>/dev/null; } + +# Kill any lingering trainer procs from a prior phase AND block until they are +# really gone, so the freed GPU VRAM is reclaimed before the next phase shards +# its embedding tables (otherwise it OOMs on the leaked memory). +# * Bracketed patterns ([t]rain_ranker, …) are REQUIRED: a plain `pkill -f +# train_ranker` issued inside `bash -lc "...train_ranker..."` matches its OWN +# command line and SIGKILLs this very shell (docker exec returns 137), which +# silently aborted the rest of the old cleanup and leaked trainers/VRAM. +# * After signalling, poll until no trainer remains (bounded), then a short +# settle so the driver finishes reclaiming device memory. +cleanup_workers() { + sx ' + for pat in "[t]rain_ranker" "[g]enerative_recommenders" "[s]pawn_main" "[m]ultiprocessing"; do + pkill -9 -f "$pat" 2>/dev/null + done + for _ in $(seq 1 30); do + pgrep -f "[t]rain_ranker" >/dev/null 2>&1 || \ + pgrep -f "[g]enerative_recommenders" >/dev/null 2>&1 || break + sleep 2 + done + sleep 3; true' || true +} +clean_ckpt() { sx "rm -rf '$1'" || true; } + +# Precheck: the dataset cache must be readable at $DATA_PATH. On B200 it must be +# staged node-local (the supervisor's stage_data_in does this) since mmap from +# virtiofs/NFS wedges; on MI350 it reads directly from NFS. Skipped when DATA_PATH +# is empty (the trainer falls back to its gin default and we don't know the path). +precheck_data() { + [[ -z "$DATA_PATH" ]] && { echo "[$(date)] data path unset — trainer will use its gin default; skipping precheck"; return 0; } + local ok + ok=$(sx "[ -d '$DATA_PATH/$DATASET_SUBDIR' ] && echo yes || echo no") + if [[ "$ok" != "yes" ]]; then + echo "FAIL: dataset cache not found at $DATA_PATH/$DATASET_SUBDIR inside '$CONTAINER' (platform=$PLATFORM)." + if [[ "$PLATFORM" == "b200" ]]; then + echo " B200: stage it node-local first (the e2e supervisor does this via stage_data_in)," + echo " or pass --data-path to an already-staged local mirror. mmap from virtiofs/NFS wedges." + else + echo " MI350/MI355: pass --data-path to the NFS dataset root (gin default is /apps/chcai/dlrm_data)." + fi + exit 1 + fi +} + +# Wait (host-side grep on the shared-NFS log) for a target regex OR a crash +# sentinel. 0=target found, 1=crash first, 2=timeout. +wait_for_log() { + local log="$LOG_DIR/$1.log"; local target_re="$2"; local timeout_s="${3:-1800}" + local elapsed=0 + while (( elapsed < timeout_s )); do + grep -qE "$target_re" "$log" 2>/dev/null && return 0 + grep -qE "Traceback|RuntimeError|OutOfMemoryError|CUDA error" "$log" 2>/dev/null && return 1 + sleep 5; elapsed=$((elapsed + 5)) + done + return 2 +} + +# Launch one trainer phase (detached), appending a PHASE_EXIT sentinel after the +# trainer returns (clean OR crash) — exactly like the production supervisor. The +# common env (data path, mode, start_ts, barrier debug) is fixed; per-phase knobs +# are passed as additional "K=V" words. +run_phase() { + local name="$1"; shift + local log="$LOG_DIR/${name}.log" + # `$*` (joined into ONE word), NOT `$@`: embedded mid-string in the + # double-quoted `bash -lc "..."`, `$@` would expand to multiple args and + # bash -lc would stop after the first override (silent 0-byte log). + local env_overrides="$*" + # Inject DLRM_DATA_PATH only when a path is set; an empty DATA_PATH means + # "use the trainer's gin default" (the meta64 NFS root). + local data_env="" + [[ -n "$DATA_PATH" ]] && data_env="DLRM_DATA_PATH=$DATA_PATH" + : > "$log" + echo "[$(date)] === phase '$name' ===" + cleanup_workers + srun --jobid="$JOBID" --overlap docker exec -d "$CONTAINER" bash -lc " + cd $REPO && + $data_env \ + HSTU_HAMMER_KERNEL=TRITON \ + MODE=streaming-train-eval \ + START_TS=$START_TS \ + WINDOW_BARRIER_DEBUG=1 \ + RUN_NAME=resume_test_$name \ + LOG=$log \ + $env_overrides \ + bash scripts/launch_slurm.sh; + echo \"PHASE_EXIT=\$? \$(date '+%F %T')\" >> $log + " +} + +# Read a scalar field from a summarize-JSON. +jget() { python3 -c "import json,sys;print(json.load(open(sys.argv[1])).get(sys.argv[2]))" "$1" "$2"; } + +FAIL=0 +fail() { echo "FAIL: $*"; FAIL=1; } + +precheck_data + +# ============================================================================= +# SCENARIO: midwindow +# ============================================================================= +run_midwindow() { + echo "########## scenario: midwindow ##########" + local LAST_STEP=$NUM_TRAIN_BATCHES + if (( DIE_AT_STEP <= 0 || DIE_AT_STEP >= NUM_TRAIN_BATCHES )); then + echo "Warning: die_at_step=$DIE_AT_STEP not strictly inside window (0, $NUM_TRAIN_BATCHES)" >&2 + fi + if (( DIE_AT_STEP % IN_WINDOW_FREQ != 0 )); then + echo "Warning: die_at_step=$DIE_AT_STEP not a multiple of in_window_freq=$IN_WINDOW_FREQ; no save lands exactly at crash" >&2 + fi + + # P1 baseline + clean_ckpt "$CKPT_ROOT" + run_phase baseline \ + "NUM_TRAIN_TS=1" "EVAL_EVERY_N_WINDOWS=0" "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" "NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES" \ + "TRAIN_SPLIT_PERCENTAGE=1.0" "DIE_AT_STEP=-1" + wait_for_log baseline "PHASE_EXIT=0" "$MW_TIMEOUT"; local rc=$? + cleanup_workers + (( rc != 0 )) && { echo "FAIL: midwindow baseline didn't finish (rc=$rc)"; tail -20 "$LOG_DIR/baseline.log"; return 1; } + + # P2 interrupted + clean_ckpt "$CKPT_ROOT" + run_phase interrupt \ + "NUM_TRAIN_TS=1" "EVAL_EVERY_N_WINDOWS=0" "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" "NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES" \ + "TRAIN_SPLIT_PERCENTAGE=1.0" \ + "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" "KEEP_LAST_N=1" \ + "DIE_AT_STEP=$DIE_AT_STEP" "CKPT_PATH=$CKPT_ROOT" + wait_for_log interrupt "die_at_step=$DIE_AT_STEP hit" "$MW_TIMEOUT"; rc=$? + cleanup_workers + (( rc != 0 )) && { echo "FAIL: interrupt didn't hit die_at_step (rc=$rc)"; tail -20 "$LOG_DIR/interrupt.log"; return 1; } + echo "Saved checkpoints after interrupt: $(sx "ls '$CKPT_ROOT' 2>/dev/null | tr '\n' ' '")" + + # P3 resume + run_phase resume \ + "NUM_TRAIN_TS=1" "EVAL_EVERY_N_WINDOWS=0" "METRIC_LOG_FREQ=1" \ + "NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES" "NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES" \ + "TRAIN_SPLIT_PERCENTAGE=1.0" \ + "IN_WINDOW_CKPT_FREQ=$IN_WINDOW_FREQ" "KEEP_LAST_N=1" \ + "DIE_AT_STEP=-1" "CKPT_PATH=$CKPT_ROOT" + # PHASE_EXIT=0 only after the (blocking) end-of-window save renames cleanly, + # so this also confirms the final atomic save completed. + wait_for_log resume "PHASE_EXIT=0" "$MW_TIMEOUT"; rc=$? + cleanup_workers + (( rc != 0 )) && { echo "FAIL: resume didn't finish (rc=$rc)"; tail -20 "$LOG_DIR/resume.log"; return 1; } + + # HARD functional invariants (deterministic; the real correctness proof). + if ! grep -qE "Resuming mid-window at train_ts=[0-9]+ batch_idx_in_window=$DIE_AT_STEP\b" "$LOG_DIR/resume.log" 2>/dev/null; then + fail "resume did not re-enter mid-window at batch_idx_in_window=$DIE_AT_STEP" + grep -E "Resuming" "$LOG_DIR/resume.log" 2>/dev/null | head -2 + fi + local rng_restored + rng_restored=$(grep -c "RNG state restored from" "$LOG_DIR/resume.log" 2>/dev/null || echo 0) + echo "RNG state restored on $rng_restored ranks" + (( rng_restored < 1 )) && fail "no RNG state restored on resume" + local first_resumed + first_resumed=$(grep -oE 'train - Step [0-9]+ metrics: \{.metric' "$LOG_DIR/resume.log" 2>/dev/null \ + | grep -oE 'Step [0-9]+' | awk '{print $2}' | sort -n | head -1) + echo "First resumed train step: $first_resumed (expect $((DIE_AT_STEP + 1)))" + [[ "$first_resumed" != "$((DIE_AT_STEP + 1))" ]] && fail "resume did not continue at step $((DIE_AT_STEP + 1)) (got $first_resumed)" + + # On-disk: atomic save + retention. + local num_ckpt stale_ckpt + num_ckpt=$(sx "ls '$CKPT_ROOT' 2>/dev/null | grep -E '^[0-9]+$' | wc -l" | tr -d ' ') + stale_ckpt=$(sx "ls '$CKPT_ROOT' 2>/dev/null | grep -E '\\.(tmp|old|staging)$' | wc -l" | tr -d ' ') + echo "Final: $num_ckpt numeric ckpt subdirs, $stale_ckpt stale dirs (expect 1, 0)" + [[ "$num_ckpt" != "1" ]] && fail "keep_last_n=1 violated (got $num_ckpt)" + [[ "$stale_ckpt" != "0" ]] && fail "stale .tmp/.old/.staging dirs left behind ($stale_ckpt)" + + # Trajectory closeness (loose sanity bound, NOT bit-equality). + python3 "$PYHELPER" parse "$LOG_DIR/baseline.log" "$LOG_DIR/traj_baseline.json" + python3 "$PYHELPER" parse "$LOG_DIR/resume.log" "$LOG_DIR/traj_resumed.json" + if ! python3 "$PYHELPER" compare "$LOG_DIR/traj_baseline.json" "$LOG_DIR/traj_resumed.json" \ + --min-resume-step $((DIE_AT_STEP + 1)) --atol "$ATOL"; then + fail "trajectory diverged beyond $ATOL (likely wrong data slice / unrestored state)" + fi + (( FAIL == 0 )) && echo "=== midwindow: PASS ===" || echo "=== midwindow: FAIL ===" +} + +# ============================================================================= +# SCENARIO: multiwindow (regression guard for the broadcast + barrier fixes) +# ============================================================================= +# Common split contract — MUST be byte-identical between mw_seed and mw_resume, +# else the resume aborts on a split-contract mismatch (the holdout_ts default of +# start_ts+num_train_ts differs between a 1-window seed and an MW_TS resume, so +# it is PINNED here). +MW_SPLIT_ENV=( "TRAIN_SPLIT_PERCENTAGE=$MW_SPLIT" "SPLIT_SALT=0" + "EVAL_HOLDOUT_TS=$MW_HOLDOUT_TS" "EVAL_HOLDOUT_NUM_WINDOWS=1" ) + +run_multiwindow() { + echo "########## scenario: multiwindow ##########" + local sum + + # P1 mw_baseline — cold multi-window run with data-pct eval. + clean_ckpt "$CKPT_ROOT" + run_phase mw_baseline \ + "NUM_TRAIN_TS=$MW_TS" "NUM_TRAIN_BATCHES=$MW_BATCHES" "NUM_EVAL_BATCHES=$MW_EVAL_BATCHES" \ + "EVAL_EVERY_N_WINDOWS=0" "EVAL_EVERY_DATA_PCT=$MW_EVAL_PCT" "METRIC_LOG_FREQ=1" \ + "${MW_SPLIT_ENV[@]}" + wait_for_log mw_baseline "PHASE_EXIT=" "$MW_RUN_TIMEOUT"; local rc=$? + cleanup_workers + (( rc == 1 )) && { echo "FAIL: mw_baseline crashed"; tail -30 "$LOG_DIR/mw_baseline.log"; return 1; } + (( rc == 2 )) && { echo "FAIL: mw_baseline timed out (possible boundary deadlock)"; tail -30 "$LOG_DIR/mw_baseline.log"; return 1; } + + sum="$LOG_DIR/mw_baseline.summary.json" + python3 "$PYHELPER" summarize "$LOG_DIR/mw_baseline.log" "$sum" >/dev/null + echo "--- mw_baseline summary ---"; cat "$sum" + local exit_code anchors barriers dpct_setup dpct_trig + exit_code=$(jget "$sum" phase_exit) + anchors=$(jget "$sum" total_train_anchors_calls) + barriers=$(jget "$sum" window_barrier_count) + dpct_setup=$(jget "$sum" data_pct_eval_setup) + dpct_trig=$(jget "$sum" data_pct_eval_trigger_count) + # (barrier B) ran through ALL windows and exited 0 — no boundary deadlock. + [[ "$exit_code" != "0" ]] && fail "mw_baseline did not complete cleanly (phase_exit=$exit_code)" + [[ "$barriers" != "$MW_TS" ]] && fail "window barrier fired $barriers times, expected $MW_TS (one per window; need world_size>=2)" + # (broadcast A) total_train_anchors computed exactly once (rank 0), not Nx. + # It is computed at loop SETUP (before any training), so this exercises the + # broadcast regardless of whether an eval later fires. + [[ "$anchors" != "1" ]] && fail "total_train_anchors computed $anchors times, expected 1 (rank-0 broadcast regressed)" + # data-fraction eval cadence set up (the path that needs total_train_anchors). + [[ "$dpct_setup" != "True" ]] && fail "data-pct eval cadence not set up (total_train_anchors path not reached)" + # Trigger firing depends on (full-window) anchor count vs the few test steps, + # so it is informational — not required to exercise the broadcast fix. + echo "data-pct eval triggers fired: $dpct_trig (informational)" + + # P2 mw_seed — 1 window → clean WINDOW_COMPLETE checkpoint. + clean_ckpt "$CKPT_ROOT" + run_phase mw_seed \ + "NUM_TRAIN_TS=1" "NUM_TRAIN_BATCHES=$MW_BATCHES" "NUM_EVAL_BATCHES=$MW_EVAL_BATCHES" \ + "EVAL_EVERY_N_WINDOWS=0" "EVAL_EVERY_DATA_PCT=$MW_EVAL_PCT" "METRIC_LOG_FREQ=1" \ + "KEEP_LAST_N=1" "CKPT_PATH=$CKPT_ROOT" "${MW_SPLIT_ENV[@]}" + wait_for_log mw_seed "PHASE_EXIT=0" "$MW_RUN_TIMEOUT"; rc=$? + cleanup_workers + (( rc != 0 )) && { echo "FAIL: mw_seed didn't finish/checkpoint (rc=$rc)"; tail -30 "$LOG_DIR/mw_seed.log"; return 1; } + local seed_ckpt + seed_ckpt=$(sx "ls '$CKPT_ROOT' 2>/dev/null | grep -E '^[0-9]+$' | sort -n | tail -1" | tr -d ' ') + echo "mw_seed end-of-window checkpoint: ${seed_ckpt:-} (expect $START_TS)" + [[ "$seed_ckpt" != "$START_TS" ]] && { fail "mw_seed did not save end-of-window ckpt $START_TS (got '$seed_ckpt')"; return 1; } + + # P3 mw_resume — relaunch over MW_TS windows; resume past the completed + # window and CROSS the boundary into the remaining windows (the exact case + # that used to deadlock). The full split contract matches the seed. + run_phase mw_resume \ + "NUM_TRAIN_TS=$MW_TS" "NUM_TRAIN_BATCHES=$MW_BATCHES" "NUM_EVAL_BATCHES=$MW_EVAL_BATCHES" \ + "EVAL_EVERY_N_WINDOWS=0" "EVAL_EVERY_DATA_PCT=$MW_EVAL_PCT" "METRIC_LOG_FREQ=1" \ + "KEEP_LAST_N=1" "CKPT_PATH=$CKPT_ROOT" "${MW_SPLIT_ENV[@]}" + wait_for_log mw_resume "PHASE_EXIT=" "$MW_RUN_TIMEOUT"; rc=$? + cleanup_workers + (( rc == 1 )) && { echo "FAIL: mw_resume crashed"; tail -30 "$LOG_DIR/mw_resume.log"; return 1; } + (( rc == 2 )) && { echo "FAIL: mw_resume timed out (possible boundary deadlock on resume)"; tail -30 "$LOG_DIR/mw_resume.log"; return 1; } + + sum="$LOG_DIR/mw_resume.summary.json" + python3 "$PYHELPER" summarize "$LOG_DIR/mw_resume.log" "$sum" >/dev/null + echo "--- mw_resume summary ---"; cat "$sum" + local r_exit r_resume_ts r_anchors r_barriers r_dpct + r_exit=$(jget "$sum" phase_exit) + r_resume_ts=$(jget "$sum" resume_completed_ts) + r_anchors=$(jget "$sum" total_train_anchors_calls) + r_barriers=$(jget "$sum" window_barrier_count) + r_dpct=$(jget "$sum" data_pct_eval_setup) + # Resumed from the completed seed window (advanced past the boundary). + [[ "$r_resume_ts" != "$START_TS" ]] && fail "mw_resume did not resume from completed train_ts=$START_TS (got $r_resume_ts)" + # Crossed the boundary into the remaining MW_TS-1 windows and exited 0. + [[ "$r_exit" != "0" ]] && fail "mw_resume did not complete cleanly (phase_exit=$r_exit) — boundary deadlock on resume?" + [[ "$r_barriers" != "$((MW_TS - 1))" ]] && fail "mw_resume barrier fired $r_barriers times, expected $((MW_TS - 1)) (windows after the resumed one)" + # Broadcast still once on the resume path; data-pct cadence rebuilt. + [[ "$r_anchors" != "1" ]] && fail "mw_resume total_train_anchors computed $r_anchors times, expected 1" + [[ "$r_dpct" != "True" ]] && fail "mw_resume data-pct eval cadence not set up" + + (( FAIL == 0 )) && echo "=== multiwindow: PASS ===" || echo "=== multiwindow: FAIL ===" +} + +# ============================================================================= +[[ "$SCENARIO" == "midwindow" || "$SCENARIO" == "all" ]] && run_midwindow +[[ "$SCENARIO" == "multiwindow" || "$SCENARIO" == "all" ]] && run_multiwindow + +if [[ "$KEEP" != "1" ]]; then + rm -rf "$LOG_DIR" + clean_ckpt "$CKPT_ROOT" +fi + +if (( FAIL == 0 )); then + echo "=== PASS: all selected scenarios validated ===" + exit 0 +fi +echo "=== FAIL: one or more scenarios failed (see above) ===" +exit 1 diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/train_test.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/train_test.py new file mode 100644 index 000000000..dfd58f6e5 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/tests/train_test.py @@ -0,0 +1,29 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +import unittest + +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.dlrm_v3.train.train_ranker import main + + +class DLRMV3TrainTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_e2e(self) -> None: + main() + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py new file mode 100644 index 000000000..a57153f60 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py @@ -0,0 +1,370 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +import argparse +import logging + +logging.basicConfig(level=logging.INFO) +import os +import sys +import traceback + +import gin +import torch +from torch import multiprocessing as mp +from torchrec.test_utils import get_free_port + +# NOTE: heavy imports of generative_recommenders.dlrm_v3.* are deferred to +# inside _main_func so that gin-driven env-var bootstrap (see +# _env_bootstrap.apply_env_bootstrap) can run BEFORE the triton kernel +# modules evaluate their `@triton.autotune` decorators at module-load time. + +logger: logging.Logger = logging.getLogger(__name__) + + +SUPPORTED_CONFIGS = { + "debug": "debug.gin", + "kuairand-1k": "kuairand_1k.gin", + "movielens-1m": "movielens_1m.gin", + "movielens-20m": "movielens_20m.gin", + "movielens-13b": "movielens_13b.gin", + "movielens-18b": "movielens_18b.gin", + "streaming-400m": "streaming_400m.gin", + "streaming-200b": "streaming_200b.gin", + "streaming-100b": "streaming_100b.gin", + "yambda-5b": "yambda_5b.gin", +} + + +def _main_func( + local_rank: int, + world_size: int, + node_rank: int, + gpus_per_node: int, + master_addr: str, + master_port: int, + gin_file: str, + mode: str, +) -> None: + # `local_rank` is the index handed out by mp.start_processes (0..gpus_per_node-1) + # and indexes this node's GPUs. The GLOBAL rank is what every downstream + # consumer wants (data sharding via StreamingWindowSampler, checkpoint I/O, + # metrics), so derive it once and pass it through as `rank`. Only the CUDA + # device must be node-local. Single-node (node_rank=0) → rank == local_rank, + # exactly as before. + rank = node_rank * gpus_per_node + local_rank + device = torch.device(f"cuda:{local_rank}") + logger.info( + f"rank: {rank} (node_rank={node_rank} local_rank={local_rank}), " + f"world_size: {world_size}, device: {device}" + ) + # Phase 1: parse gin early with skip_unknown=True so env-bootstrap + # bindings take effect BEFORE any module-level @gin.configurable + # discovers itself. This is required because triton @triton.autotune + # decorators in generative_recommenders.ops.triton.* read env vars at + # module import time, and the heavy imports below pull those in. + from generative_recommenders.dlrm_v3.train._env_bootstrap import apply_env_bootstrap + from generative_recommenders.dlrm_v3.train.mlperf_logging_utils import ( + get_mlperf_logger, + mlperf_checkpoint_present, + ) + + gin.parse_config_file(gin_file, skip_unknown=True) + apply_env_bootstrap() + + # Cold-start vs resume, decided from the on-disk checkpoint BEFORE setup so + # the one-time INIT/RUN markers fire on a genuine cold start only and are NOT + # re-emitted on a resume relaunch — the MLPerf run (run_start..run_stop) spans + # the resume as a single coherent event stream in one appended log file. + mlperf_resume = mlperf_checkpoint_present(os.environ.get("CKPT_PATH", "")) + # Rank-0-gated MLPerf logger, only for the streaming-train-eval path. `fresh` + # truncates the log on cold start (one run_start per file) but appends on a + # resume so the pre-crash events are preserved and continued. + mlperf_logger = ( + get_mlperf_logger(rank=rank, fresh=not mlperf_resume) + if mode == "streaming-train-eval" + else None + ) + # INIT_START fires before setup on a cold start only (resume continues the + # already-open run, whose markers were emitted by the original process). + mlperf_cold_start = mlperf_logger is not None and not mlperf_resume + if mlperf_cold_start: + mlperf_logger.event(key=mlperf_logger.constants.CACHE_CLEAR, value=True) + mlperf_logger.start(key=mlperf_logger.constants.INIT_START) + + # Phase 2: heavy imports. Triton kernel modules evaluate their autotune + # decorators here, using the env vars set above. + from generative_recommenders.dlrm_v3.checkpoint import load_dmp_checkpoint + from generative_recommenders.dlrm_v3.train.utils import ( + cleanup, + decorrelate_runtime_rng, + eval_loop, + make_model, + make_optimizer_and_shard, + make_train_test_dataloaders, + seed_everything, + setup, + streaming_train_eval_loop, + train_eval_loop, + train_loop, + ) + from generative_recommenders.dlrm_v3.utils import ( + MetricsLogger, + get_gpu_peak_flops, + ) + + setup( + rank=rank, + world_size=world_size, + master_addr=master_addr, + master_port=master_port, + device=device, + ) + # Phase 3: re-parse to bind the @gin.configurables now that they are + # registered. The earlier skip_unknown pass already consumed the + # env-bootstrap binding, but bindings are idempotent so re-applying is + # fine, and this pass is the one that actually wires up make_model, + # make_train_test_dataloaders, etc. + gin.parse_config_file(gin_file) + + # Seed all RNGs (gin-configurable $SEED) BEFORE make_model() so weight init + # is reproducible run-to-run. Must follow the full parse above so the binding + # is wired, and precede make_model() below. + seed_everything(rank=rank) + + model, model_configs, embedding_table_configs = make_model() + model, optimizer = make_optimizer_and_shard( + model=model, + device=device, + world_size=world_size, + local_world_size=gpus_per_node, + embedding_table_configs=embedding_table_configs, + ) + # Decorrelate forward-time stochasticity (HSTU dropout) per data-parallel + # rank. MUST run after make_model() + make_optimizer_and_shard() so the + # replicated dense weights and sharded embeddings stay init-identical across + # ranks; this only offsets the global RNG by rank so dropout masks differ. + decorrelate_runtime_rng(rank=rank) + train_dataloader, test_dataloader = make_train_test_dataloaders( + hstu_config=model_configs, + embedding_table_configs=embedding_table_configs, + ) + # TFLOPS/MFU reporting: query the model's static dense estimate + + # current GPU's peak FLOPS. Both default to 0 if the model doesn't + # expose get_num_flops_per_sample, in which case MetricsLogger silently + # drops the tflops fields from the perf line. + inner_model = model.module if hasattr(model, "module") else model + num_flops_per_sample = ( + float(inner_model.get_num_flops_per_sample()) + if hasattr(inner_model, "get_num_flops_per_sample") + else 0.0 + ) + gpu_peak_flops = get_gpu_peak_flops( + "bf16" if getattr(model_configs, "bf16_training", True) else "fp32" + ) + # Streaming fixed-holdout eval uses the dual fresh/cumulative metric sets: + # window_* = fresh per-pass full-holdout, lifetime_* = cumulative across + # passes (AUC via O(bins) histogram). Other modes keep the legacy single set. + metrics = MetricsLogger( + multitask_configs=model_configs.multitask_configs, + batch_size=train_dataloader.batch_size, + window_size=2500, + device=device, + rank=rank, + # Pass the live world_size so metric normalization is correct at any + # node count; the gin's MetricsLogger.world_size default (=8) is only a + # single-node fallback and would mis-normalize a multi-node run. + world_size=world_size, + num_flops_per_sample=num_flops_per_sample, + gpu_peak_flops=gpu_peak_flops, + model=model, + eval_cumulative=(mode == "streaming-train-eval"), + # Lifetime-AUC backend + bins/window come from gin (see yambda_5b.gin: + # MetricsLogger.{train,eval}_lifetime_auc_mode / cumulative_auc_bins / + # lifetime_auc_window), env-overridable. eval_cumulative stays explicit + # because it is runtime-mode dependent, not a config knob. + ) + # Capture streaming resume hint (None for cold start / non-streaming + # checkpoints). For the streaming-train-eval mode, we forward this into + # streaming_train_eval_loop so it can advance past the last completed + # window OR re-enter the partial window and skip already-trained batches. + resume_train_ts, resume_batch_idx_in_window, resume_split_contract, resume_cold_start = ( + load_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metrics, + device=device, + rank=rank, + ) + ) + + # MLPerf run markers: open the run exactly once. On a cold start emit + # submission info + hyperparameters + INIT_STOP/RUN_START and mark the run as + # started (persisted in the checkpoint via metrics.mlperf_run_started). On a + # resume, load_dmp_checkpoint restored mlperf_run_started=True, so we skip the + # markers and just continue the stream. `metrics.mlperf_run_started` guards a + # double-emit even if cold/resume detection and the checkpoint ever disagree. + if mlperf_cold_start and not metrics.mlperf_run_started: + # Submission info + hyperparameters + INIT_STOP/RUN_START, all emitted by + # the logger (optimizer names/LRs read from gin internally). Seed is the + # value setup() resolved and exported to $SEED. + mlperf_logger.log_run_start( + global_batch_size=world_size * int(train_dataloader.batch_size), + seed=int(os.environ.get("SEED", "1")), + ) + metrics.mlperf_run_started = True + # Pass the logger to the loop whenever MLPerf logging is enabled, so block / + # eval / train_loss / run_stop events emit on BOTH a cold start and a resume. + mlperf_run_active = mlperf_logger is not None and metrics.mlperf_run_started + + # train loop + try: + if mode == "train": + train_loop( + rank=rank, + model=model, + dataloader=train_dataloader, + optimizer=optimizer, + metric_logger=metrics, + device=device, + ) + elif mode == "eval": + # reinit metrics logger for eval + metrics = MetricsLogger( + multitask_configs=model_configs.multitask_configs, + batch_size=train_dataloader.batch_size, + window_size=1000, + device=device, + rank=rank, + world_size=world_size, + ) + eval_loop( + rank=rank, + model=model, + dataloader=test_dataloader, + metric_logger=metrics, + device=device, + ) + elif mode == "train-eval": + train_eval_loop( + rank=rank, + model=model, + train_dataloader=train_dataloader, + eval_dataloader=test_dataloader, + optimizer=optimizer, + metric_logger=metrics, + device=device, + ) + elif mode == "streaming-train-eval": + streaming_train_eval_loop( + rank=rank, + model=model, + optimizer=optimizer, + metric_logger=metrics, + device=device, + hstu_config=model_configs, + embedding_table_configs=embedding_table_configs, + resume_train_ts=resume_train_ts, + resume_batch_idx_in_window=resume_batch_idx_in_window, + resume_split_contract=resume_split_contract, + resume_cold_start=resume_cold_start, + # Only pass the logger when run boundaries were emitted, so the + # loop never produces orphan block/eval events. + mlperf_logger=mlperf_logger if mlperf_run_active else None, + ) + except Exception as e: + logger.info(traceback.format_exc()) + raise Exception(e) + finally: + # Graceful distributed teardown on both success and failure: barrier so + # all ranks finish in lockstep, then destroy the process group (best- + # effort) to avoid noisy TCPStore/NCCL shutdown warnings at exit. + if torch.distributed.is_initialized(): + try: + torch.distributed.barrier() + except Exception: + logger.info("teardown barrier failed (non-fatal)") + try: + cleanup() + except Exception: + logger.info("teardown destroy_process_group failed (non-fatal)") + + +def get_args(): # pyre-ignore [3] + """Parse commandline.""" + parser = argparse.ArgumentParser() + parser.add_argument( + "--dataset", default="debug", choices=SUPPORTED_CONFIGS.keys(), help="dataset" + ) + parser.add_argument( + "--mode", + default="train", + choices=["train", "eval", "train-eval", "streaming-train-eval"], + help="mode", + ) + args, unknown_args = parser.parse_known_args() + logger.warning(f"unknown_args: {unknown_args}") + return args + + +def main() -> None: + args = get_args() + logger.info(args) + assert args.dataset in SUPPORTED_CONFIGS, f"Unsupported dataset: {args.dataset}" + assert args.mode in [ + "train", + "eval", + "train-eval", + "streaming-train-eval", + ], f"Unsupported mode: {args.mode}" + # Distributed topology (single-node defaults reproduce the legacy behavior): + # GPUS_PER_NODE local procs to spawn on THIS node (default: all visible GPUs) + # NNODES/NODE_RANK multi-node fan-out, set by the SLURM launcher + # WORLD_SIZE global rank count = NNODES * GPUS_PER_NODE + # MASTER_ADDR/PORT rank-0 rendezvous; the port MUST match across nodes, so + # honor it from the env when set and only fall back to a + # random free port for the standalone single-node path. + GPUS_PER_NODE = int(os.environ.get("GPUS_PER_NODE", 0)) or torch.cuda.device_count() + NNODES = int(os.environ.get("NNODES", 1)) + NODE_RANK = int(os.environ.get("NODE_RANK", 0)) + WORLD_SIZE = NNODES * GPUS_PER_NODE + MASTER_ADDR = os.environ.get("MASTER_ADDR", "localhost") + MASTER_PORT = str(os.environ.get("MASTER_PORT") or get_free_port()) + gin_path = f"{os.path.dirname(__file__)}/gin/{SUPPORTED_CONFIGS[args.dataset]}" + logger.info( + f"launching: nnodes={NNODES} node_rank={NODE_RANK} " + f"gpus_per_node={GPUS_PER_NODE} world_size={WORLD_SIZE} " + f"master={MASTER_ADDR}:{MASTER_PORT}" + ) + + mp.start_processes( + _main_func, + args=( + WORLD_SIZE, + NODE_RANK, + GPUS_PER_NODE, + MASTER_ADDR, + MASTER_PORT, + gin_path, + args.mode, + ), + nprocs=GPUS_PER_NODE, + join=True, + start_method="spawn", + ) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py new file mode 100644 index 000000000..41811cc90 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/train/utils.py @@ -0,0 +1,3175 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-strict +import logging +import os +import threading +import time +from collections.abc import Iterator +from datetime import timedelta +from typing import ( + Any, + Callable, + Dict, + Iterable, + List, + Optional, + Set, + Tuple, + Type, + Union, +) + +import gin +import torch +import torchrec +from generative_recommenders.dlrm_v3.checkpoint import save_dmp_checkpoint, WINDOW_COMPLETE +from generative_recommenders.dlrm_v3.configs import ( + get_embedding_table_config, + get_hstu_configs, +) +from generative_recommenders.dlrm_v3.datasets.dataset import ( + collate_fn, + Dataset, + Samples, +) +from generative_recommenders.dlrm_v3.utils import get_dataset, MetricsLogger, Profiler +from generative_recommenders.common import HammerKernel +from generative_recommenders.modules.dlrm_hstu import DlrmHSTU, DlrmHSTUConfig +from torch import distributed as dist +from torch.distributed.optim import ( + _apply_optimizer_in_backward as apply_optimizer_in_backward, +) +from torch.optim.optimizer import Optimizer +from torch.utils.data import DataLoader, Dataset as TorchDataset +from torch.utils.data.distributed import _T_co, DistributedSampler +from torchrec.distributed.embedding_types import EmbeddingComputeKernel +from torchrec.distributed.model_parallel import DistributedModelParallel +from torchrec.distributed.planner import EmbeddingShardingPlanner, Topology +from torchrec.distributed.planner.types import ParameterConstraints +from torchrec.distributed.sharding_plan import get_default_sharders +from torchrec.distributed.types import ShardedTensor, ShardingEnv, ShardingType +from torchrec.modules.embedding_configs import EmbeddingConfig +from torchrec.modules.embedding_modules import ( + EmbeddingBagCollection, + EmbeddingCollection, +) +from torchrec.optim.keyed import CombinedOptimizer, KeyedOptimizerWrapper +from torchrec.optim.optimizers import in_backward_optimizer_filter +from torchrec.sparse.jagged_tensor import KeyedJaggedTensor + +logger: logging.Logger = logging.getLogger(__name__) + +TORCHREC_TYPES: Set[Type[Union[EmbeddingBagCollection, EmbeddingCollection]]] = { + EmbeddingBagCollection, + EmbeddingCollection, +} + +# Embedding placement vocabulary -> torchrec compute kernel. Used by +# make_optimizer_and_shard to translate the gin/env placement strings +# ("hbm"/"uvm"/"uvm_caching") into ParameterConstraints. "auto" (or anything not +# in this map) means "no constraint": the planner decides from the HBM cap. +_PLACEMENT_TO_KERNEL: Dict[str, EmbeddingComputeKernel] = { + "hbm": EmbeddingComputeKernel.FUSED, + "uvm": EmbeddingComputeKernel.FUSED_UVM, + "uvm_caching": EmbeddingComputeKernel.FUSED_UVM_CACHING, +} + +# Per-table sharding-type vocabulary -> torchrec ShardingType. Used by +# make_optimizer_and_shard to pin a table's shard layout via ParameterConstraints +# (e.g. move a hot, data-skewed table off ROW_WISE to COLUMN_WISE so its +# embedding all-to-all load is balanced by rank instead of routed by row/value). +# "auto" (or anything not in this map) means "no constraint": the planner decides. +# Short aliases (rw/cw/tw/twrw) are accepted alongside the canonical names. +_SHARDING_TO_TYPE: Dict[str, ShardingType] = { + "row_wise": ShardingType.ROW_WISE, + "column_wise": ShardingType.COLUMN_WISE, + "table_wise": ShardingType.TABLE_WISE, + "table_row_wise": ShardingType.TABLE_ROW_WISE, + "table_column_wise": ShardingType.TABLE_COLUMN_WISE, + "data_parallel": ShardingType.DATA_PARALLEL, + "rw": ShardingType.ROW_WISE, + "cw": ShardingType.COLUMN_WISE, + "tw": ShardingType.TABLE_WISE, + "twrw": ShardingType.TABLE_ROW_WISE, +} + + +@gin.configurable +def seed_everything(seed: int = -1, rank: int = 0) -> None: + """Seed all RNGs (same value on every rank) for reproducible dense weight init. + + Call right before make_model(), after setup() (process group needed for the + broadcast) and the gin parse. seed < 0 ($SEED unset) draws a fresh random seed + per run (rank 0 broadcasts; exported to $SEED); seed >= 0 reproduces a run. + Data order/split are independent of this seed (StreamingWindowSampler/$SPLIT_SALT). + """ + import random + + import numpy as np + + pinned = seed >= 0 + if not pinned: + # rank 0 draws a random seed and broadcasts it so all ranks agree (an + # identical seed on every rank is REQUIRED, else dense weight init + # diverges across ranks and DDP/AllReduce trains garbage). + if dist.is_available() and dist.is_initialized(): + device = torch.device(f"cuda:{torch.cuda.current_device()}") + drawn = int.from_bytes(os.urandom(4), "little") if rank == 0 else 0 + _seed_t = torch.tensor([drawn], dtype=torch.int64, device=device) + dist.broadcast(_seed_t, src=0) + seed = int(_seed_t.item()) + else: + seed = int.from_bytes(os.urandom(4), "little") + # Export the resolved value so the run is reproducible after the fact. + os.environ["SEED"] = str(seed) + + logger.info( + f"[rank {rank}] seeding all RNGs with SEED={seed} " + f"({'pinned via $SEED' if pinned else 'random per-run; set $SEED to reproduce'})" + ) + random.seed(seed) + np.random.seed(seed) + torch.manual_seed(seed) + torch.cuda.manual_seed_all(seed) + + +@gin.configurable +def decorrelate_runtime_rng(rank: int = 0, enabled: bool = True) -> None: + """Re-seed torch/cuda with $SEED + rank so HSTU dropout draws different masks + per data-parallel rank (seed_everything's identical seed would draw the same). + + MUST run after make_model() + make_optimizer_and_shard() so init stays + identical across ranks; it perturbs only forward-time stochasticity. + Reproducible (pure fn of $SEED + rank; RNG state checkpointed). enabled=False + keeps the legacy identical-mask behavior. + """ + if not enabled: + logger.info( + f"[rank {rank}] decorrelate_runtime_rng disabled; dropout masks " + f"identical across ranks" + ) + return + base = int(os.environ.get("SEED", "1")) + offset_seed = base + int(rank) + torch.manual_seed(offset_seed) + torch.cuda.manual_seed_all(offset_seed) + logger.info( + f"[rank {rank}] decorrelated runtime RNG: SEED={base} + rank={rank} " + f"=> {offset_seed} (per-rank dropout masks)" + ) + + +def setup( + rank: int, + world_size: int, + master_port: int, + device: torch.device, + master_addr: str = "localhost", +) -> dist.ProcessGroup: + # Default "localhost" keeps the single-node path unchanged; multi-node + # launches pass the rank-0 host so every node rendezvouses at the same addr. + os.environ["MASTER_ADDR"] = master_addr + os.environ["MASTER_PORT"] = str(master_port) + + BACKEND = dist.Backend.NCCL + # Process-group / NCCL watchdog timeout (seconds). Env-overridable so a + # diagnostic run can use a short, finite timeout that trips the NCCL flight + # recorder dump (TORCH_NCCL_TRACE_BUFFER_SIZE + TORCH_NCCL_DUMP_ON_TIMEOUT) + # on a collective desync instead of hanging for the full default. + TIMEOUT = int(os.environ.get("PG_TIMEOUT_S", "1800")) + + # set device BEFORE init_process_group so NCCL binds this rank to its + # own GPU; otherwise every rank's first CUDA context lands on GPU 0, + # leaving stale allocations and triggering OOMs on rank 0. + torch.cuda.set_device(device) + + # NOTE: RNG seeding for reproducible weight init lives in seed_everything(), + # which train_ranker calls right before make_model() (after setup() so the + # process group is initialized for the cross-rank seed broadcast, and after + # the full gin parse so the gin-configurable $SEED is bound). Seeding here + # would be too early to be gin-configurable. + + # initialize the process group + # + # The default PG timeout must match TIMEOUT (not the 600s NCCL default): + # checkpoint saves go through DCP collectives on *this* default PG, and the + # 560GB sparse-embedding write is both slow on shared NFS and badly skewed + # across ranks (shards range ~37GB..~95GB), so the fastest rank can sit in + # the post-write allgather/barrier well past 600s waiting for the slowest + # rank. The stock 600s watchdog then SIGABRTs an otherwise-healthy job. + if not dist.is_initialized(): + dist.init_process_group( + "nccl", + rank=rank, + world_size=world_size, + device_id=device, + timeout=timedelta(seconds=TIMEOUT), + ) + + pg = dist.new_group( + backend=BACKEND, + timeout=timedelta(seconds=TIMEOUT), + ) + + return pg + + +def cleanup() -> None: + dist.destroy_process_group() + + +def _window_boundary_barrier( + device: torch.device, world_size: int, train_ts: int +) -> None: + """Collective rendezvous at a streaming window boundary. + + The per-window data prep (``window_indices``: an O(N) mask over the ~18 GB + mmap'd ``anchor_ts`` array) can complete at very different times across + ranks. The embedding input-dist all-to-all that follows is a collective, so + if a fast rank reaches it while a slow rank is still in prep, the NCCL + stream desyncs and the job deadlocks (one rank a collective behind the + rest). Synchronizing here makes prep-time skew harmless: every rank waits + until all ranks have a ready window before any issues the first forward. + + Cost is one near-zero-payload barrier per window (299 total over a full + run). In the healthy case prep already overlapped the previous window's + compute via the prefetcher, so the barrier returns immediately; it only + blocks for the real prep skew it is there to absorb. + """ + if not (dist.is_available() and dist.is_initialized()) or world_size <= 1: + return + t0 = time.time() + if device.type == "cuda": + dist.barrier(device_ids=[device.index]) + else: + dist.barrier() + waited = time.time() - t0 + # Surface non-trivial skew (the thing this barrier exists to absorb) so a + # node with a slow rank is visible without trawling the flight recorder. + if waited > 5.0: + logger.warning( + "[window-barrier] train_ts=%d: waited %.1fs at boundary " + "rendezvous (per-rank data-prep skew)", + train_ts, + waited, + ) + # Test/debug observability: the healthy-path barrier is otherwise SILENT + # (the skew warning above only fires on >5s waits), so the resume e2e test + # has no signal that the boundary rendezvous actually executed. When + # WINDOW_BARRIER_DEBUG=1, rank 0 emits exactly one line per crossed window + # so the test can assert the barrier ran at EVERY boundary (regression guard + # for the desync deadlock the barrier fixes). Off by default — zero prod cost. + if os.environ.get("WINDOW_BARRIER_DEBUG") == "1" and dist.get_rank() == 0: + logger.info( + "[window-barrier] train_ts=%d rendezvous complete (waited %.3fs)", + train_ts, + waited, + ) + + +class HammerToTorchDataset(TorchDataset): + def __init__( + self, + dataset: Dataset, + ) -> None: + self.dataset: Dataset = dataset + + def __getitem__(self, idx: int) -> Tuple[KeyedJaggedTensor, KeyedJaggedTensor]: + self.dataset.load_query_samples([idx]) + sample = self.dataset.get_sample(idx) + self.dataset.unload_query_samples([idx]) + return sample + + def __getitems__( + self, indices: List[int] + ) -> List[Tuple[KeyedJaggedTensor, KeyedJaggedTensor]]: + self.dataset.load_query_samples(indices) + samples = [self.dataset.get_sample(i) for i in indices] + self.dataset.unload_query_samples(indices) + return samples + + +class _ChainedRanges: + """O(1) __len__ + O(log K) __getitem__ over a sequence of `range`s. + + Lets `torch.utils.data.Subset(dataset, _ChainedRanges([r1, r2, ...]))` + avoid materializing a Python list of all per-block indices (which at + multi-billion totals is ~28 B/int and dominates host RAM). + """ + + def __init__(self, ranges: List[range]) -> None: + self._ranges: List[range] = list(ranges) + offsets = [0] + for r in self._ranges: + offsets.append(offsets[-1] + len(r)) + self._offsets: List[int] = offsets + + def __len__(self) -> int: + return self._offsets[-1] + + def __getitem__(self, idx: int) -> int: + import bisect + if idx < 0: + idx += self._offsets[-1] + if idx < 0 or idx >= self._offsets[-1]: + raise IndexError(idx) + bucket = bisect.bisect_right(self._offsets, idx) - 1 + return self._ranges[bucket][idx - self._offsets[bucket]] + + +class ChunkDistributedSampler(DistributedSampler[_T_co]): + """ + Each rank reads a contiguous chunk (trunk) of the input data + """ + + def __init__( + self, + dataset: TorchDataset, + num_replicas: Optional[int] = None, + rank: Optional[int] = None, + shuffle: bool = True, + seed: int = 1, + drop_last: bool = False, + ) -> None: + super().__init__( + dataset=dataset, + num_replicas=num_replicas, + rank=rank, + shuffle=shuffle, + seed=seed, + drop_last=drop_last, + ) + + def __iter__(self) -> Iterator[_T_co]: + if self.shuffle: + g = torch.Generator() + g.manual_seed(self.seed + self.epoch * 1001 + int(self.rank)) + indices_t = torch.randperm(self.num_samples, generator=g) + else: + indices_t = torch.arange(self.num_samples) + assert self.drop_last is True, ( + "drop_last must be True for ChunkDistributedSampler" + ) + indices_t = indices_t + (self.num_samples * int(self.rank)) + assert indices_t.numel() == self.num_samples + # Iterate via the numpy view, NOT directly over the tensor: iter(Tensor) + # calls Tensor.unbind(0) which eagerly materializes one zero-dim Tensor + # object per element (~600 B each). For 40 M eval / 525 M train samples + # that's 24 GB / 315 GB of [heap] growth per rank, blowing host RAM + # before the first batch. numpy's iter yields one Python int at a time + # with O(1) extra memory. + indices_np = indices_t.numpy() + return (int(x) for x in indices_np) + + def set_epoch(self, epoch: int) -> None: + logger.warning(f"Setting epoch to {epoch}") + self.epoch = epoch + + +@gin.configurable +def make_model( + dataset: str, + bf16_training: bool = False, + hammer_kernel: Optional[str] = None, +) -> Tuple[torch.nn.Module, DlrmHSTUConfig, Dict[str, EmbeddingConfig]]: + hstu_config = get_hstu_configs(dataset) + table_config = get_embedding_table_config(dataset) + + # bf16 autocast is off by default: on the PYTORCH attn backend the + # pt_hstu_attention QK einsum backward overflows in bf16 at long + # sequences (NaN at step 1 when N>1k). Safe with TRITON; flip via + # `make_model.bf16_training = True` in the gin. + model = DlrmHSTU( + hstu_configs=hstu_config, + embedding_tables=table_config, + is_inference=False, + bf16_training=bf16_training, + ) + + # HSTU attention/compute kernel backend. Precedence: + # HSTU_HAMMER_KERNEL env var > make_model.hammer_kernel gin > model default. + # The env var stays as an ad-hoc override (e.g. forcing PYTORCH for a one-off + # debug run) without editing the gin. Note: the fused TRITON path avoids + # materializing the dense [B, H, N, N] attention-score tensor that the PYTORCH + # path allocates (~32 GiB at N=2048, bs=1024), so TRITON is both faster and + # far lighter on HBM. On older ROCm, TRITON could hit PassManager errors at + # some shapes (make_ttgir) — fall back to PYTORCH via the gin/env if so. + kernel_choice = ( + os.environ.get("HSTU_HAMMER_KERNEL", "").upper() + or (hammer_kernel.upper() if hammer_kernel else "") + ) + if kernel_choice: + model.set_hammer_kernel(HammerKernel[kernel_choice]) + logger.warning(f"HSTU hammer kernel set to: {kernel_choice}") + + return ( + model, + hstu_config, + table_config, + ) + + +@gin.configurable() +def dense_optimizer_factory_and_class( + optimizer_name: str, + betas: Tuple[float, float], + eps: float, + weight_decay: float, + momentum: float, + learning_rate: float, +) -> Tuple[ + Type[Optimizer], Dict[str, Any], Callable[[Iterable[torch.Tensor]], Optimizer] +]: + kwargs: Dict[str, Any] = {"lr": learning_rate} + if optimizer_name == "Adam": + optimizer_cls = torch.optim.Adam + kwargs.update({"betas": betas, "eps": eps, "weight_decay": weight_decay}) + elif optimizer_name == "SGD": + optimizer_cls = torch.optim.SGD + kwargs.update({"weight_decay": weight_decay, "momentum": momentum}) + elif optimizer_name == "AdamW": + optimizer_cls = torch.optim.AdamW + kwargs.update({"betas": betas, "eps": eps, "weight_decay": weight_decay}) + else: + raise Exception("Unsupported optimizer!") + + optimizer_factory = lambda params: optimizer_cls(params, **kwargs) + + logger.info( + f"[dense optimizer] {optimizer_name} learning_rate={learning_rate} " + f"(resolved from gin; override via $DENSE_LR)" + ) + return optimizer_cls, kwargs, optimizer_factory + + +@gin.configurable() +def sparse_optimizer_factory_and_class( + optimizer_name: str, + betas: Tuple[float, float], + eps: float, + weight_decay: float, + momentum: float, + learning_rate: float, +) -> Tuple[ + Type[Optimizer], Dict[str, Any], Callable[[Iterable[torch.Tensor]], Optimizer] +]: + kwargs: Dict[str, Any] = {"lr": learning_rate} + if optimizer_name == "Adam": + optimizer_cls = torch.optim.Adam + beta1, beta2 = betas + kwargs.update( + {"beta1": beta1, "beta2": beta2, "eps": eps, "weight_decay": weight_decay} + ) + elif optimizer_name == "SGD": + optimizer_cls = torchrec.optim.SGD + kwargs.update({"weight_decay": weight_decay, "momentum": momentum}) + elif optimizer_name == "RowWiseAdagrad": + optimizer_cls = torchrec.optim.RowWiseAdagrad + beta1, beta2 = betas + kwargs.update( + { + "eps": eps, + "beta1": beta1, + "beta2": beta2, + "weight_decay": weight_decay, + } + ) + else: + raise Exception("Unsupported optimizer!") + + optimizer_factory = lambda params: optimizer_cls(params, **kwargs) + + logger.info( + f"[sparse optimizer] {optimizer_name} learning_rate={learning_rate} " + f"(resolved from gin; override via $SPARSE_LR)" + ) + return optimizer_cls, kwargs, optimizer_factory + + +_FBGEMM_LOWMEM_PATCHED = False + + +def _patch_fbgemm_lowmem_clamp_cast(enabled: bool = True, rank0: bool = False) -> None: + """Replace fbgemm's quant clamp+cast with a memory-frugal equivalent. + + ``enabled`` is the gin/env-driven kill switch (see + ``make_optimizer_and_shard.qcomm_lowmem_clamp_cast`` / + ``$QCOMM_LOWMEM_CODEC``). Default ON; pass ``enabled=False`` to fall back to + stock fbgemm (e.g. to reproduce the pre-patch OOM, or if a future fbgemm + version changes the codec and the patch needs revalidation). + + fbgemm's ``fp32_to_fp16_with_clamp`` (and the bf16 variant) does + ``torch.clamp(tensor, MIN, MAX).half()``. ``torch.clamp(...)`` allocates a + SECOND full-size fp32 tensor (same numel as the input) *before* the cast, so + the transient peak is input(fp32) + clamp_temp(fp32) + output(fp16) ~= 2.5x + the input. On a skewed row-wise-sharded batch the hottest shard's embedding + tensor is huge (observed 81.5 GiB clamp temp), and that extra fp32 copy is + exactly the allocation that OOMs the rank — which then exits the train loop + while peers block forever in the a2a (a 30-min NCCL-watchdog hang). See + HANG_ROOTCAUSE.md / flight-recorder dump for the full diagnosis. + + Cast FIRST then clamp IN PLACE: ``tensor.half().clamp_(MIN, MAX)``. This + allocates only the fp16 output (no full-size fp32 temp), cutting the peak by + the size of the input tensor, while being numerically identical: an fp32 + value above HALF_MAX casts to +inf, which clamp_ maps back to HALF_MAX (and + NaNs pass through unchanged), matching clamp-then-cast bit for bit. Safe to + do in place because the codec ``encode()`` runs inside the qcomm autograd + ``Function.forward`` (grad disabled), so there is no graph to corrupt. + """ + global _FBGEMM_LOWMEM_PATCHED + if not enabled: + if rank0: + logger.warning( + "[qcomm-lowmem] DISABLED (qcomm_lowmem_clamp_cast=False / " + "QCOMM_LOWMEM_CODEC=0) — running stock fbgemm clamp+cast, which " + "allocates a full-size fp32 clamp temp and can OOM->hang the " + "hottest row-wise embedding shard on skewed batches." + ) + return + if _FBGEMM_LOWMEM_PATCHED: + return + try: + from fbgemm_gpu import quantize_comm, quantize_utils + + _HMIN = quantize_utils.TORCH_HALF_MIN + _HMAX = quantize_utils.TORCH_HALF_MAX + _BMIN = quantize_utils.TORCH_BFLOAT16_MIN + _BMAX = quantize_utils.TORCH_BFLOAT16_MAX + + def _lowmem_fp16(tensor: torch.Tensor) -> torch.Tensor: + return tensor.half().clamp_(_HMIN, _HMAX) + + def _lowmem_bf16(tensor: torch.Tensor) -> torch.Tensor: + return tensor.bfloat16().clamp_(_BMIN, _BMAX) + + # Patch BOTH the definition module and quantize_comm, which imported the + # names directly (``from .quantize_utils import fp32_to_fp16_with_clamp``) + # so its module-level reference must be overridden too. + for _mod in (quantize_utils, quantize_comm): + if hasattr(_mod, "fp32_to_fp16_with_clamp"): + _mod.fp32_to_fp16_with_clamp = _lowmem_fp16 + if hasattr(_mod, "fp32_to_bf16_with_clamp"): + _mod.fp32_to_bf16_with_clamp = _lowmem_bf16 + + _FBGEMM_LOWMEM_PATCHED = True + if rank0: + logger.info( + "[qcomm-lowmem] patched fbgemm fp32->fp16/bf16 clamp+cast to " + "cast-then-clamp_ (drops the full-size fp32 clamp temp; avoids " + "OOM on skewed row-wise embedding a2a)" + ) + except Exception as e: # noqa: BLE001 — patch is best-effort, never fatal + if rank0: + logger.warning( + "[qcomm-lowmem] could not patch fbgemm clamp+cast (%s: %s); " + "running with stock (higher-peak) quantizer", + type(e).__name__, + e, + ) + + +def _maybe_apply_qcomm_a2a( + sharders: List[Any], + device: torch.device, + forward_precision: str = "fp32", + backward_precision: str = "fp32", + lowmem_clamp_cast: bool = True, +) -> List[Any]: + """Optionally quantize the embedding all-to-all payload via TorchRec qcomm. + + The yambda-5b embedding shuffle is the dominant, bandwidth-bound (multi-node) + collective (~14.5 GB/rank fp32); a bf16/fp16 wire dtype halves it. Quant/ + dequant happen inside the comm op, transparent to the lookup consumer. Ported + from the DLRMv2 R2 lever, retargeted from ``EmbeddingBagCollectionSharder`` to + the sequence ``EmbeddingCollectionSharder`` this model uses. + + Forward and backward are configured independently because they have different + numerical needs (TorchRec golden_training/train_dlrm.py recommends + forward=fp16, backward=bf16): the forward carries bounded embedding + activations where fp16's extra mantissa helps, while gradients have a wider + range that can overflow fp16, so bf16 (fp32 exponent range) is safer there. + bf16 and fp16 are both 2 bytes, so the wire volume / perf is identical — the + choice is purely numerical. + + Args (set via gin on ``make_optimizer_and_shard``, env-overridable). Each is + one of ``fp32`` (that direction unquantized) | ``bf16`` | ``fp16``. If BOTH + are fp32 the sharders are returned untouched (identical to baseline trunk). + """ + _COMM = {"bf16": "BF16", "fp16": "FP16", "fp32": "FP32"} + fwd = (forward_precision or "fp32").strip().lower() + bwd = (backward_precision or "fp32").strip().lower() + rank0 = (not dist.is_initialized()) or dist.get_rank() == 0 + for name, p in (("forward", fwd), ("backward", bwd)): + if p not in _COMM: + # Misconfigured precision: fail loudly rather than silently running + # fp32. A typo in SPARSE_A2A_{FWD,BWD} must not pass as "no quant". + raise ValueError( + f"DLRMV4 qcomm a2a: unknown {name} precision {p!r} " + f"(want one of fp32|bf16|fp16)" + ) + if fwd == "fp32" and bwd == "fp32": + return sharders + # Before building the codec, swap fbgemm's clamp+cast for a memory-frugal + # equivalent — see _patch_fbgemm_lowmem_clamp_cast for why (avoids a full + # extra fp32 temp that OOMs the hottest row-wise shard on skewed batches). + # Gated by `lowmem_clamp_cast` (gin/env); ON by default. + _patch_fbgemm_lowmem_clamp_cast(enabled=lowmem_clamp_cast, rank0=rank0) + try: + from torchrec.distributed.embedding import EmbeddingCollectionSharder + from torchrec.distributed.fbgemm_qcomm_codec import ( + CommType, + get_qcomm_codecs_registry, + QCommsConfig, + ) + + qcfg = QCommsConfig( + forward_precision=getattr(CommType, _COMM[fwd]), + backward_precision=getattr(CommType, _COMM[bwd]), + ) + registry = get_qcomm_codecs_registry(qcfg, device=device) + except Exception as e: # noqa: BLE001 + # A configured quantized a2a that fails to build is a hard error. Silently + # downgrading to fp32 would change numerics/throughput with no signal, and + # a partial failure (one rank fp32, others fp16) would also desync the + # collectives. Raise on every rank so the whole job aborts consistently. + raise RuntimeError( + f"DLRMV4 qcomm a2a: failed to enable configured quantization " + f"(forward={fwd} backward={bwd}): {type(e).__name__}: {e}" + ) from e + + new_sharders = [] + replaced = False + for s in sharders: + if type(s).__name__ == "EmbeddingCollectionSharder" and not replaced: + new_sharders.append( + EmbeddingCollectionSharder(qcomm_codecs_registry=registry) + ) + replaced = True + else: + new_sharders.append(s) + if not replaced: + # Codec registry built fine, but there was no EmbeddingCollectionSharder to + # bind it to, so the quantized a2a would be silently inert. Treat this as a + # hard failure too — "configured but not applied" is the bug we want caught. + raise RuntimeError( + f"DLRMV4 qcomm a2a: quantization configured (forward={fwd} " + f"backward={bwd}) but no EmbeddingCollectionSharder was found to attach " + f"the qcomm codec registry to; refusing to run with quantization " + f"silently disabled" + ) + if rank0: + logger.info( + "DLRMV4 qcomm a2a ENABLED: forward=%s backward=%s " + "replaced_ec_sharder=%s", + fwd, + bwd, + replaced, + ) + return new_sharders + + +def _maybe_apply_ec_index_dedup( + sharders: List[Any], + enabled: bool, +) -> List[Any]: + """Optionally enable TorchRec's in-batch index dedup on the EC sharder. + + Runs as a FINAL pass AFTER get_default_sharders() and _maybe_apply_qcomm_a2a, + so it covers BOTH sharder construction paths with a single knob: + * qcomm ON (SPARSE_A2A_* != fp32, the default): the EC sharder was already + rebuilt with a qcomm codec registry inside _maybe_apply_qcomm_a2a. + * qcomm OFF (SPARSE_A2A_* == fp32): that function early-returns the bare + get_default_sharders() list untouched. + In either case we rebuild the one EmbeddingCollectionSharder in place, + PRESERVING its existing fused_params + qcomm codec registry, and only flip + use_index_dedup. That ordering guarantees dedup composes with (never clobbers) + the fp16/bf16 a2a instead of depending on which qcomm branch ran. + + use_index_dedup collapses duplicate ids within a batch to unique rows before + the input all-to-all (torch.ops.fbgemm.jagged_unique_indices), does the + embedding lookup once, then scatters back via reverse_indices. For a + non-pooled EmbeddingCollection (this sequence model) that reconstruction is + exact, so the forward is numerically lossless — unlike the fp16 a2a it does + not perturb AUC/NE. The buffers it registers (_hash_size_cumsum_tensor_*, + _hash_size_offset_tensor_*) are persistent=False, so it does NOT change the + on-disk shard layout and can be toggled on an existing checkpoint with no + reshard. Payoff scales with the in-batch duplicate rate: large under + user-major batching (STREAMING_SHUFFLE_FRACTION=0, the default, where one + user's UIH re-reads the same item/artist/album ids), shrinking toward + neutral/negative as batch diversity rises. + + The knob only exists on EmbeddingCollectionSharder, which is the sole default + sharder this model actually binds (a pure sequence EmbeddingCollection), so + no other sharder needs touching. Configured via gin on + ``make_optimizer_and_shard`` (env-overridable $EC_INDEX_DEDUP). + """ + if not enabled: + return sharders + + from torchrec.distributed.embedding import EmbeddingCollectionSharder + + rank0 = (not dist.is_initialized()) or dist.get_rank() == 0 + new_sharders = [] + replaced = False + for s in sharders: + if type(s).__name__ == "EmbeddingCollectionSharder" and not replaced: + new_sharders.append( + EmbeddingCollectionSharder( + fused_params=s.fused_params, + qcomm_codecs_registry=s.qcomm_codecs_registry, + use_index_dedup=True, + ) + ) + replaced = True + else: + new_sharders.append(s) + if not replaced: + # Dedup configured but no EC sharder to bind it to would mean it is + # silently inert — the same "configured but not applied" bug class the + # qcomm path guards against. Fail loudly on every rank instead. + raise RuntimeError( + "DLRMV4 EC index dedup configured (use_index_dedup=True) but no " + "EmbeddingCollectionSharder was found to enable it on; refusing to " + "run with dedup silently disabled" + ) + if rank0: + logger.info("DLRMV4 EC index dedup ENABLED (use_index_dedup=True)") + return new_sharders + + +def _embedding_table_names( + model: torch.nn.Module, + embedding_table_configs: Optional[Dict[str, EmbeddingConfig]], +) -> List[str]: + """All embedding table names the planner will place. + + Prefers the authoritative `embedding_table_configs` (keys == table names == + planner parameter names). Falls back to walking the model's EBC/EC modules + when the configs are not passed in. + """ + if embedding_table_configs: + return list(embedding_table_configs.keys()) + names: List[str] = [] + for _, module in model.named_modules(): + if type(module) in TORCHREC_TYPES: + if isinstance(module, EmbeddingBagCollection): + names.extend(c.name for c in module.embedding_bag_configs()) + elif isinstance(module, EmbeddingCollection): + names.extend(c.name for c in module.embedding_configs()) + return names + + +def _build_placement_constraints( + model: torch.nn.Module, + embedding_placement: str, + embedding_placement_overrides: Dict[str, str], + embedding_table_configs: Optional[Dict[str, EmbeddingConfig]], + embedding_sharding_overrides: Optional[Dict[str, str]] = None, +) -> Dict[str, ParameterConstraints]: + """Translate gin/env placement + sharding strings into ParameterConstraints. + + Two orthogonal per-table knobs are merged into one constraint per table: + + * Placement (compute kernel / memory tier): + ``embedding_placement_overrides.get(name, embedding_placement)``. + ``auto``/empty -> no compute-kernel constraint (planner decides from HBM). + * Sharding type (shard layout): ``embedding_sharding_overrides.get(name)``. + ``auto``/empty (or absent) -> no sharding-type constraint (planner decides, + which is ROW_WISE for the large yambda tables). Use e.g. ``column_wise`` + to move a hot, data-skewed table off ROW_WISE so its embedding all-to-all + is balanced by rank instead of routed by (hot) row. + + A table is added to the returned dict only if at least one knob is set for it + (so with everything ``auto`` we return {} and the plan is byte-identical to + the legacy path). Unknown values raise ValueError. + """ + embedding_sharding_overrides = embedding_sharding_overrides or {} + valid_place = set(_PLACEMENT_TO_KERNEL) | {"auto", ""} + for where, val in [ + ("embedding_placement", embedding_placement), + *[ + (f"embedding_placement_overrides[{k}]", v) + for k, v in embedding_placement_overrides.items() + ], + ]: + if val not in valid_place: + raise ValueError( + f"Invalid embedding placement {val!r} for {where}; " + f"expected one of {sorted(valid_place - {''})}." + ) + valid_shard = set(_SHARDING_TO_TYPE) | {"auto", ""} + for k, v in embedding_sharding_overrides.items(): + if v not in valid_shard: + raise ValueError( + f"Invalid embedding sharding {v!r} for " + f"embedding_sharding_overrides[{k}]; " + f"expected one of {sorted(valid_shard - {''})}." + ) + + names = _embedding_table_names(model, embedding_table_configs) + unknown = ( + set(embedding_placement_overrides) | set(embedding_sharding_overrides) + ) - set(names) + if unknown: + logger.warning( + "[emb-placement] override(s) for unknown table(s) %s ignored; " + "known tables: %s", + sorted(unknown), + sorted(names), + ) + + constraints: Dict[str, ParameterConstraints] = {} + resolved_place: Dict[str, str] = {} + resolved_shard: Dict[str, str] = {} + for name in names: + placement = embedding_placement_overrides.get(name, embedding_placement) + sharding = embedding_sharding_overrides.get(name, "auto") + resolved_place[name] = placement or "auto" + resolved_shard[name] = sharding or "auto" + kernel = _PLACEMENT_TO_KERNEL.get(placement) + stype = _SHARDING_TO_TYPE.get(sharding) + kwargs: Dict[str, Any] = {} + if kernel is not None: + kwargs["compute_kernels"] = [kernel.value] + if stype is not None: + kwargs["sharding_types"] = [stype.value] + if kwargs: + constraints[name] = ParameterConstraints(**kwargs) + + rank = dist.get_rank() if dist.is_initialized() else 0 + if rank == 0: + logger.info( + "[emb-placement] placement(global=%r overrides=%s) sharding(overrides=%s) " + "-> resolved_placement=%s resolved_sharding=%s " + "(constrained=%d/%d tables; the rest are planner-auto)", + embedding_placement, + embedding_placement_overrides or {}, + embedding_sharding_overrides or {}, + resolved_place, + resolved_shard, + len(constraints), + len(names), + ) + return constraints + + +@gin.configurable +def make_optimizer_and_shard( + model: torch.nn.Module, + device: torch.device, + world_size: int, + local_world_size: Optional[int] = None, + hbm_cap_gb: int = 260, + sparse_a2a_forward_precision: str = "fp32", + sparse_a2a_backward_precision: str = "fp32", + qcomm_lowmem_clamp_cast: bool = True, + ec_index_dedup: bool = False, + embedding_placement: str = "auto", + embedding_placement_overrides: Optional[Dict[str, str]] = None, + embedding_sharding_overrides: Optional[Dict[str, str]] = None, + embedding_table_configs: Optional[Dict[str, EmbeddingConfig]] = None, +) -> Tuple[DistributedModelParallel, torch.optim.Optimizer]: + dense_opt_cls, dense_opt_args, dense_opt_factory = ( + dense_optimizer_factory_and_class() + ) + + sparse_opt_cls, sparse_opt_args, sparse_opt_factory = ( + sparse_optimizer_factory_and_class() + ) + # Fuse sparse optimizer to backward step + for k, module in model.named_modules(): + if type(module) in TORCHREC_TYPES: + for _, param in module.named_parameters(prefix=k): + if param.requires_grad: + apply_optimizer_in_backward( + sparse_opt_cls, [param], sparse_opt_args + ) + sharders = get_default_sharders() + sharders = _maybe_apply_qcomm_a2a( + sharders, + device, + forward_precision=sparse_a2a_forward_precision, + backward_precision=sparse_a2a_backward_precision, + lowmem_clamp_cast=qcomm_lowmem_clamp_cast, + ) + # Final pass: enable in-batch index dedup on the EC sharder regardless of + # whether the qcomm branch above rebuilt it (fp16 default) or returned the + # bare get_default_sharders() list (fp32). Preserves the qcomm registry. + sharders = _maybe_apply_ec_index_dedup(sharders, enabled=ec_index_dedup) + # local_world_size = GPUs per node so the planner respects the intra-node + # (xGMI/NVLink) vs inter-node hierarchy when placing shards. Defaults to + # world_size for the single-node case (no behavior change). + logger.info( + "[hbm-cap] make_optimizer_and_shard: hbm_cap_gb=%s (planner Topology hbm_cap=%d bytes), " + "world_size=%s local_world_size=%s", + hbm_cap_gb, + hbm_cap_gb * 1024 * 1024 * 1024, + world_size, + local_world_size or world_size, + ) + # Resolve per-table embedding placement (gin/env-driven). Global default + # `embedding_placement` applies to every table; `embedding_placement_overrides` + # (table name -> placement) wins per table. Tables resolving to "auto" carry + # no constraint (planner decides from hbm_cap). When nothing is constrained we + # pass constraints=None so the plan is byte-identical to the legacy path. + constraints = _build_placement_constraints( + model=model, + embedding_placement=embedding_placement, + embedding_placement_overrides=embedding_placement_overrides or {}, + embedding_sharding_overrides=embedding_sharding_overrides or {}, + embedding_table_configs=embedding_table_configs, + ) + planner = EmbeddingShardingPlanner( + topology=Topology( + local_world_size=local_world_size or world_size, + world_size=world_size, + compute_device="cuda", + hbm_cap=hbm_cap_gb * 1024 * 1024 * 1024, + ddr_cap=0, + ), + constraints=constraints or None, + ) + pg = dist.GroupMember.WORLD + env = ShardingEnv.from_process_group(pg) # pyre-ignore [6] + pg = env.process_group + + plan = planner.collective_plan(model, sharders, pg) + + # Authoritative placement log: report the compute kernel the planner ACTUALLY + # assigned to each table (vs the [emb-placement] line above, which reports what + # was requested). "fused" = HBM; "fused_uvm"/"fused_uvm_caching" = UVM-backed. + # Rank 0 only, best-effort (never break the build over a logging shape change). + if (dist.get_rank() if dist.is_initialized() else 0) == 0: + try: + for module_path, param_plans in plan.plan.items(): + for param_name, ps in param_plans.items(): + logger.info( + "[emb-placement] plan: %s.%s -> compute_kernel=%s " + "sharding_type=%s", + module_path, + param_name, + getattr(ps, "compute_kernel", "?"), + getattr(ps, "sharding_type", "?"), + ) + except Exception as e: # logging only; must never fail the build + logger.warning("[emb-placement] could not dump plan kernels: %s", e) + + # Re-seed right before DMP materializes/inits the sharded embedding tables. + # The per-table seeded init_fn (configs.get_embedding_table_config) handles + # the eager path, but the fused FBGEMM TBE path inits weights on-device and + # may bypass init_fn, drawing from the global RNG instead. Re-seeding here + # (same value on every rank) makes embedding init reproducible run-to-run for + # a fixed sharding plan (Tier 1). Dense params are already initialized in + # make_model, so this does not perturb them. + _emb_seed = int(os.environ.get("SEED", "1")) + torch.manual_seed(_emb_seed) + torch.cuda.manual_seed_all(_emb_seed) + logger.info(f"[emb-init] re-seeded RNGs before DMP with SEED={_emb_seed}") + + # Shard model + model = DistributedModelParallel( + module=model, + device=device, + plan=plan, + sharders=sharders, + ) + + # --- startup init checksum (reproducibility probe) ------------------------- + # Right after DMP materializes real weights, log a deterministic fingerprint + # of every parameter so two builds with the same $SEED + sharding plan can be + # diffed for byte-level init reproducibility. For sharded embeddings we + # all-reduce the per-shard (count, sum, sumsq) so the fingerprint covers the + # WHOLE table independent of how rows split across ranks; replicated dense + # params use rank 0's local copy. + # OFF BY DEFAULT: the fp64 reductions below (.sum(dtype=float64) / + # vector_norm(dtype=float64)) materialize a full fp64 copy of each local + # embedding shard (~2x the fp32 shard, i.e. >150 GiB for the big tables), + # which leaves almost no HBM headroom after sharding and will OOM the build on + # any node with residual memory. Only enable for explicit reproducibility + # checks, ideally with a smaller batch / on a clean node. Enable with + # INIT_CHECKSUM=1. + if os.environ.get("INIT_CHECKSUM", "0") == "1": + import hashlib + + _rank = dist.get_rank() if dist.is_initialized() else 0 + _fps: List[str] = [] + for _name, _p in sorted(model.named_parameters(), key=lambda kv: kv[0]): + _sharded = isinstance(_p, ShardedTensor) + if _sharded: + _shards = _p.local_shards() + _loc = _shards[0].tensor if _shards else None + else: + _loc = _p + if _loc is None or _loc.numel() == 0: + _cnt, _sm, _sq = 0.0, 0.0, 0.0 + else: + _det = _loc.detach() + _cnt = float(_det.numel()) + _sm = _det.sum(dtype=torch.float64).item() + _nrm = torch.linalg.vector_norm( + _det, ord=2, dtype=torch.float64 + ).item() + _sq = _nrm * _nrm + if _sharded and dist.is_initialized(): + _stat = torch.tensor( + [_cnt, _sm, _sq], dtype=torch.float64, device=device + ) + dist.all_reduce(_stat, op=dist.ReduceOp.SUM) + _cnt, _sm, _sq = _stat.tolist() + _fps.append(f"{_name}|{int(_cnt)}|{_sm:.6f}|{_sq:.6f}") + if _rank == 0: + logger.info( + f"[init-checksum] {'sharded' if _sharded else 'dense'} " + f"{_name} n={int(_cnt)} sum={_sm:.6f} sumsq={_sq:.6f}" + ) + if _rank == 0: + _digest = hashlib.sha256("\n".join(_fps).encode()).hexdigest()[:16] + logger.info( + f"[init-checksum] SEED={os.environ.get('SEED', '?')} " + f"params={len(_fps)} digest={_digest}" + ) + + # Create keyed optimizer + all_optimizers = [] + all_params = {} + non_fused_sparse_params = {} + for k, v in in_backward_optimizer_filter(model.named_parameters()): + if v.requires_grad: + if isinstance(v, ShardedTensor): + non_fused_sparse_params[k] = v + else: + all_params[k] = v + + if non_fused_sparse_params: + all_optimizers.append( + ( + "sparse_non_fused", + KeyedOptimizerWrapper( + params=non_fused_sparse_params, optim_factory=sparse_opt_factory + ), + ) + ) + + if all_params: + all_optimizers.append( + ( + "dense", + KeyedOptimizerWrapper( + params=all_params, + optim_factory=dense_opt_factory, + ), + ) + ) + output_optimizer = CombinedOptimizer(all_optimizers) + output_optimizer.init_state(set(model.sparse_grad_parameter_names())) + return model, output_optimizer + + +@gin.configurable +def make_streaming_dataloader( + dataset: HammerToTorchDataset, + ts: Optional[int] = None, + batch_size: int = 0, + num_workers: int = 0, + prefetch_factor: int = 0, + train_only: bool = False, + indices: Optional["np.ndarray"] = None, +) -> DataLoader: + # `indices` (explicit anchor index array) is used by the eval path to + # iterate the FIXED user-holdout set, which spans a window range rather than + # a single ts. Otherwise restrict to window `ts`; train_only=True drops + # held-out eval users so the non-persistent TRAIN loader never trains on + # them (no-leakage guarantee). + if indices is not None: + dataset.dataset.set_active_indices(indices) # pyre-ignore [16] + else: + assert ts is not None, "make_streaming_dataloader needs ts or indices" + dataset.dataset.set_ts(ts, train_only=train_only) # pyre-ignore [16] + total_items = dataset.dataset.get_item_count() + subset = torch.utils.data.Subset(dataset, range(total_items)) + # shuffle=False keeps temporal order within the window: a non-shuffling + # DistributedSampler hands rank r the strided slice indices[r::num_replicas] + # (round-robin), so all ranks stay on the same time front and consume the + # window in index order. Fork ctx mirrors the train path (COW-share the + # mmap'd store instead of pickling it into every worker). + mp_ctx = "fork" if num_workers and num_workers > 0 else None + dataloader = DataLoader( + dataset=subset, + batch_size=batch_size, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, + num_workers=num_workers, + prefetch_factor=prefetch_factor, + sampler=DistributedSampler(subset, shuffle=False, drop_last=True), + multiprocessing_context=mp_ctx, + ) + return dataloader + + +class StreamingWindowSampler(torch.utils.data.Sampler): + """Per-rank sampler whose index list is swapped each window. + + Yields this rank's round-robin slice of the active window's GLOBAL anchor + indices (into the dataset's ``_positions``). Because indices are global, a + single DataLoader with ``persistent_workers=True`` can be reused across all + windows: the main process re-iterates this sampler each window and ships the + new indices to the already-forked workers, which map any global index via + the shared mmap. No per-window worker respawn / dataset re-pickle. + + Round-robin striding (rank r gets ``indices[r::world_size]``) over the + time-sorted window keeps every rank on the same time front; the window is + truncated to a multiple of ``world_size`` so all ranks get equal counts + (required for DDP collective lockstep). + """ + + def __init__(self, rank: int, world_size: int) -> None: + self._rank: int = rank + self._world_size: int = world_size + self._indices: List[int] = [] + + def set_window(self, global_indices, skip_samples: int = 0) -> None: + """Install this window's per-rank index list, optionally fast-forwarding. + + ``skip_samples`` drops the first N per-rank samples from the list so the + next ``__iter__`` starts at sample N+1 in this rank's slice. Used on + resume to skip batches that were already trained: pass + ``skip_samples = batch_size * batches_completed`` and the dataloader + emits batches starting at exactly the next unseen batch. + + The skip is safe because the sample order is fully deterministic given + (global_indices, rank, world_size): we re-derive the same per-rank list + as the pre-crash run, just hand back a tail slice of it. + """ + n = (len(global_indices) // self._world_size) * self._world_size + per_rank = global_indices[:n][self._rank :: self._world_size].tolist() + if skip_samples < 0 or skip_samples > len(per_rank): + raise ValueError( + f"skip_samples={skip_samples} out of [0, {len(per_rank)}] " + f"for rank={self._rank} world_size={self._world_size}" + ) + self._indices = per_rank[skip_samples:] + + def __iter__(self): + return iter(self._indices) + + def __len__(self) -> int: + return len(self._indices) + + +@gin.configurable +def make_persistent_streaming_dataloader( + dataset: HammerToTorchDataset, + sampler: StreamingWindowSampler, + batch_size: int, + num_workers: int, + prefetch_factor: int, +) -> DataLoader: + """One reusable DataLoader for the whole streaming run. ``sampler`` is + mutated per window via ``set_window``; workers persist across windows.""" + use_workers = bool(num_workers and num_workers > 0) + return DataLoader( + dataset=dataset, + batch_size=batch_size, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, + num_workers=num_workers, + prefetch_factor=prefetch_factor if use_workers else None, + sampler=sampler, + persistent_workers=use_workers, + multiprocessing_context="fork" if use_workers else None, + ) + + +class _PrefetchingWindowLoader: + """Double-buffered window loader for the persistent streaming path. + + Holds ``n_buffers`` pre-forked persistent worker pools that ping-pong: while + the current window trains on one pool, the *next* window's index selection + (``window_indices``) and first-batch prefetch are prepared on another pool + in a background thread. By the time training advances, that window is warm, + so the per-window reset (mask + first-batch stall) is hidden behind GPU + compute (~0 dead time at the boundary). + + Worker pools are forked once on the main thread at the start of ``stream``; + afterwards only iterator resets happen (no forks), so background-thread + preparation cannot fork while other threads hold locks. + """ + + def __init__( + self, + dataset: "HammerToTorchDataset", + sampler_factory, + dl_factory, + n_buffers: int = 2, + ) -> None: + self._dataset = dataset + self._n = n_buffers + self._samplers = [sampler_factory() for _ in range(n_buffers)] + self._dls = [dl_factory(s) for s in self._samplers] + self._iters: List[Optional[object]] = [None] * n_buffers + + def _prepare(self, buf: int, ts: int, skip_samples: int = 0) -> None: + # train_window_indices() is the O(N) mask (+ uid-hash filter for the + # holdout); numpy releases the GIL for it, so it overlaps the main + # thread's GPU dispatch. iter() then kicks off this pool's background + # prefetch. This is a TRAIN-only loader, so held-out eval users are + # excluded here. `skip_samples` is non-zero only for the very first + # window after a mid-window resume; subsequent windows always start at 0. + self._samplers[buf].set_window( + self._dataset.dataset.train_window_indices(ts), skip_samples=skip_samples + ) + self._iters[buf] = iter(self._dls[buf]) + + def stream(self, ts_list: List[int], first_skip_samples: int = 0): + """Stream (ts, iterator) pairs. `first_skip_samples` is applied ONLY to + the first ts in ``ts_list`` (the mid-window-resumed window); every + subsequent window starts at sample 0 of its own per-rank list.""" + n = len(ts_list) + if n == 0: + return + threads: List[Optional[threading.Thread]] = [None] * self._n + # Prime the first n_buffers windows on the main thread (forks all pools). + for b in range(min(self._n, n)): + skip = first_skip_samples if b == 0 else 0 + self._prepare(b, ts_list[b], skip_samples=skip) + for i in range(n): + buf = i % self._n + if threads[buf] is not None: + threads[buf].join() + threads[buf] = None + yield ts_list[i], self._iters[buf] + # This pool is now free; prefetch the window n_buffers ahead. + # No skip on subsequent windows — only the first prepared window + # carries `first_skip_samples`. + j = i + self._n + if j < n: + th = threading.Thread( + target=self._prepare, args=(buf, ts_list[j]), daemon=True + ) + th.start() + threads[buf] = th + + +@gin.configurable +def make_train_test_dataloaders( + batch_size: int, + dataset_type: str, + hstu_config: DlrmHSTUConfig, + train_split_percentage: float, + embedding_table_configs: Dict[str, EmbeddingConfig], + new_path_prefix: str = "", + num_workers: int = 0, + num_blocks: int = 1, + prefetch_factor: Optional[int] = None, + eval_batch_size: Optional[int] = None, +) -> Tuple[DataLoader, DataLoader]: + dataset_class, kwargs = get_dataset( + name=dataset_type, new_path_prefix=new_path_prefix + ) + kwargs["embedding_config"] = embedding_table_configs + + # Create dataset + dataset = HammerToTorchDataset( + dataset=dataset_class(hstu_config=hstu_config, is_inference=False, **kwargs) + ) + total_items = dataset.dataset.get_item_count() + items_per_block = total_items // num_blocks + train_size_per_block = round(train_split_percentage * items_per_block) + # Avoid `extend(range(...))` which materializes a Python list of all sample + # indices — at 3.2B yambda samples × 28 bytes/int ≈ 90 GB/rank just for + # train_inds. Subset accepts any sequence with O(1) __len__ and __getitem__, + # so pass range objects (or a tiny chained view) directly. + if num_blocks == 1: + train_inds = range(0, train_size_per_block) + test_inds = range(train_size_per_block, items_per_block) + else: + train_inds = _ChainedRanges([ + range(i * items_per_block, i * items_per_block + train_size_per_block) + for i in range(num_blocks) + ]) + test_inds = _ChainedRanges([ + range(i * items_per_block + train_size_per_block, (i + 1) * items_per_block) + for i in range(num_blocks) + ]) + train_set = torch.utils.data.Subset(dataset, train_inds) + test_set = torch.utils.data.Subset(dataset, test_inds) + + # When the parent rank is started via mp.start_processes(start_method="spawn"), + # torch.multiprocessing's default Process context is also "spawn". DataLoader + # then pickles `self._dataset` to send to each worker — which for our mmap'd + # 211 GB yambda store materializes the entire dataset into the parent's anon + # memory (~230 GB/rank). Forcing "fork" lets workers inherit the parent's + # mmap'd pages via COW with zero extra anon. + mp_ctx = "fork" if num_workers and num_workers > 0 else None + train_dataloader = DataLoader( + dataset=train_set, + batch_size=batch_size, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, + num_workers=num_workers, + prefetch_factor=prefetch_factor, + sampler=ChunkDistributedSampler(train_set, drop_last=True, shuffle=True), + multiprocessing_context=mp_ctx, + ) + test_dataloader = DataLoader( + dataset=test_set, + batch_size=eval_batch_size if eval_batch_size is not None else batch_size, + shuffle=False, + collate_fn=collate_fn, + drop_last=True, + num_workers=num_workers, + prefetch_factor=prefetch_factor, + sampler=ChunkDistributedSampler(test_set, drop_last=True, shuffle=True), + multiprocessing_context=mp_ctx, + ) + return train_dataloader, test_dataloader + + +# THROWAWAY DIAG state: per-embedding-table lookup stats accumulated across a +# fixed window of steps (DIAG_EMB_STEPS, default 100) so the reported numbers are +# averaged/aggregated rather than a single noisy batch. Rank-0 only. +_EMB_DIAG_ACC: Dict[str, Dict[str, Any]] = {} +_EMB_DIAG_NBATCH: int = 0 +# Cap (per-batch lookups) below which we also track a TRUE global-unique set +# across the whole window (cheap for contextual/cross tables, total == batch +# size). Sequential tables (item/artist/album) blow past this and only get the +# per-batch averages. +_EMB_DIAG_GLOBAL_CAP: int = 1 << 17 # 131072 + + +def _log_unique_embedding_diag( + sample, rank: int, step: int, max_steps: int = 100, log_every: int = 50 +) -> None: + """Diagnostic: aggregate per-embedding-table lookup stats over a step window. + + Quantifies the user-major batching concern — when consecutive sliding-window + anchors come from the same few users, a batch reads very few UNIQUE embedding + rows (low unique/total), so embedding lookups are highly redundant. Covers the + base id tables AND the cross-feature tables (user_x_* / *_x_hour hashed + combos); the user_x_* tables should be the most redundant under shuffle OFF. + + Accumulates over ``max_steps`` batches and emits an aggregate summary (mean + per-batch unique%/hot%/top10%, plus a true global-unique over the whole + window for the small contextual/cross tables). Rank-0 only, non-fatal. + """ + if rank != 0: + return + global _EMB_DIAG_NBATCH + try: + from generative_recommenders.dlrm_v3.configs import YAMBDA_5B_CROSS_SPECS + + cross_caps = {name: n for (name, _k, n, _s) in YAMBDA_5B_CROSS_SPECS} + + def _table_of(key: str): + # cross tables match by exact name; resolve BEFORE substring fallbacks + # so e.g. 'user_x_artist' isn't misread as the artist_id table. + if key in cross_caps: + return key, cross_caps[key] + if key == "uid" or key.endswith("_uid") or key.endswith(".uid"): + return "uid", 0 + if "artist" in key: + return "artist_id", 0 + if "album" in key: + return "album_id", 0 + if "item" in key and key.endswith("id"): + return "item_id", 0 + return None, 0 + + for tag, kjt in ( + ("uih", sample.uih_features_kjt), + ("cand", sample.candidates_features_kjt), + ): + for key in kjt.keys(): + table, cap = _table_of(key) + if table is None: + continue + vals = kjt[key].values() + total = int(vals.numel()) + if total == 0: + continue + u, counts = torch.unique(vals, return_counts=True) + uniq = int(u.numel()) + hot1 = int(counts.max().item()) + k = min(10, uniq) + topk = int(torch.topk(counts, k).values.sum().item()) + + slot = _EMB_DIAG_ACC.setdefault( + f"{tag}.{key}", + { + "table": table, + "cap": cap, + "n": 0, + "tot": 0, + "uniq": 0, + "upct": 0.0, + "upct_min": 100.0, + "upct_max": 0.0, + "hot1pct": 0.0, + "topkpct": 0.0, + "glob": None, # running global-unique id tensor (small tables) + }, + ) + upct = 100.0 * uniq / total + slot["n"] += 1 + slot["tot"] += total + slot["uniq"] += uniq + slot["upct"] += upct + slot["upct_min"] = min(slot["upct_min"], upct) + slot["upct_max"] = max(slot["upct_max"], upct) + slot["hot1pct"] += 100.0 * hot1 / total + slot["topkpct"] += 100.0 * topk / total + if total <= _EMB_DIAG_GLOBAL_CAP: + prev = slot["glob"] + merged = u if prev is None else torch.cat([prev, u]) + slot["glob"] = torch.unique(merged) + + _EMB_DIAG_NBATCH += 1 + n = _EMB_DIAG_NBATCH + if n % log_every == 0 or n >= max_steps: + lines = [f"emb-diag AGGREGATE over {n} batches (step<= {step}):"] + for name in sorted(_EMB_DIAG_ACC): + s = _EMB_DIAG_ACC[name] + c = max(1, s["n"]) + cap_s = f" cap={s['cap']/1e6:.0f}M" if s["cap"] else "" + glob_s = "" + if s["glob"] is not None: + g = int(s["glob"].numel()) + glob_s = ( + f" | global_uniq={g} over {s['tot']} seen " + f"({s['tot']/max(1,g):.1f}x reuse)" + ) + lines.append( + f" {name}[{s['table']}]{cap_s}: " + f"avg_tot={s['tot']/c:.0f} " + f"avg_uniq%={s['upct']/c:.1f} " + f"(min={s['upct_min']:.1f} max={s['upct_max']:.1f}) " + f"avg_hot1%={s['hot1pct']/c:.1f} " + f"avg_top10%={s['topkpct']/c:.1f}" + f"{glob_s}" + ) + logger.info("\n".join(lines)) + except Exception as e: # diagnostic must never break training + logger.warning(f"emb-diag failed: {e}") + + +@gin.configurable +def train_loop( + rank: int, + model: torch.nn.Module, + dataloader: torch.utils.data.DataLoader, + optimizer: Optimizer, + metric_logger: MetricsLogger, + device: torch.device, + num_epochs: int, + num_batches: Optional[int] = None, + output_trace: bool = False, + metric_log_frequency: int = 1, + checkpoint_frequency: int = 100, + start_batch_idx: int = 0, + streaming_diag_unique_emb: bool = False, + # lr_scheduler: to-do: Add a scheduler +) -> None: + model.train() + batch_idx: int = start_batch_idx + profiler = Profiler(rank) if output_trace else None + + for epoch in range(num_epochs): + dataloader.sampler.set_epoch(epoch) # pyre-ignore [16] + for sample in dataloader: + if streaming_diag_unique_emb and batch_idx < int( + os.environ.get("DIAG_EMB_STEPS", "100") + ): + _log_unique_embedding_diag( + sample, + rank, + batch_idx, + max_steps=int(os.environ.get("DIAG_EMB_STEPS", "100")), + log_every=metric_log_frequency, + ) + optimizer.zero_grad() + sample.to(device) + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + # pyre-ignore + sum(aux_losses.values()).backward() + optimizer.step() + metric_logger.update( + mode="train", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + if batch_idx % metric_log_frequency != 0: + metric_logger.compute_and_log( + mode="train", + additional_logs={ + "losses": aux_losses, + }, + ) + if batch_idx % checkpoint_frequency == 0 and batch_idx > 0: + save_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + rank=rank, + batch_idx=batch_idx, + ) + batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if num_batches is not None and batch_idx >= num_batches: + break + if num_batches is not None and batch_idx >= num_batches: + break + + +@gin.configurable +def eval_loop( + rank: int, + model: torch.nn.Module, + dataloader: torch.utils.data.DataLoader, + metric_logger: MetricsLogger, + device: torch.device, + metric_log_frequency: int = 1, + num_batches: Optional[int] = None, + output_trace: bool = False, + # lr_scheduler: to-do: Add a scheduler +) -> None: + model.eval() + # Exclude eval wall-time from the train step-time window (see _run_eval_window). + metric_logger.pause_perf("eval") + batch_idx: int = 0 + profiler = Profiler(rank) if output_trace else None + metric_logger.reset(mode="eval") + with torch.no_grad(): + for sample in dataloader: + sample.to(device) + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + metric_logger.update( + mode="eval", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + if batch_idx % metric_log_frequency != 0: + metric_logger.compute_and_log(mode="eval") + batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if num_batches is not None and batch_idx >= num_batches: + break + metric_logger.compute_and_log(mode="eval") + for k, v in metric_logger.compute(mode="eval").items(): + print(f"{k}: {v}") + metric_logger.resume_perf("eval") + + +class _PipelineModelWrapper(torch.nn.Module): + """Adapt ``DlrmHSTU.forward`` to the ``(loss, output)`` contract that + ``TrainPipelineSparseDist`` expects. + + The wrapped ``model`` is the same DMP instance handed to the pipeline as + ``model=``; the pipeline rewrites its sharded ``EmbeddingCollection`` in + place, so calling it here is what lets the embedding all-to-all overlap the + dense forward/backward compute. + """ + + def __init__(self, model: torch.nn.Module) -> None: + super().__init__() + self._model = model + + def forward( + self, batch: Samples + ) -> Tuple[torch.Tensor, Tuple[Any, ...]]: + # The model runs in `_pipeline_mode`: it takes the whole batch as its + # single arg and reads the pre-merged sparse KJT off it. This keeps the + # EmbeddingCollection input a plain getattr on the batch placeholder so + # TorchRec pipelines its input_dist (instead of skipping it for "input + # modifications"). + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = self._model(batch) + loss = sum(aux_losses.values()) + num_candidates = batch.candidates_features_kjt.lengths().view( + len(batch.candidates_features_kjt.keys()), -1 + )[0] + output = ( + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + num_candidates, + ) + return loss, output + + +def build_train_pipeline( + model: torch.nn.Module, + optimizer: Optimizer, + device: torch.device, + grad_clip_norm: float = 1.0, +) -> Any: + """Build a ``TrainPipelineSparseDist`` for the DMP-wrapped HSTU model. + + The 3-stage pipeline overlaps (1) H2D transfer of batch N+2, (2) the sparse + data-dist all-to-all of batch N+1's embedding lookup, and (3) dense fwd/bwd + of batch N, on separate CUDA streams. Requires the model to be wrapped with + ``DistributedModelParallel`` (see ``make_optimizer_and_shard``). + """ + # Lazy import: keeps module import working on torchrec builds that move or + # rename the pipeline, and matches the reference Primus-DLRM setup. + from torchrec.distributed.train_pipeline import TrainPipelineSparseDist + + # Switch the (DMP-wrapped) HSTU model into pipeline mode so both the fx trace + # and the live forward consume the batch as a single arg and read the + # pre-merged sparse KJT off it — required for the embedding input_dist to be + # pipelined. Eval call sites pass the batch the same way (see train_eval_loop). + underlying = model.module if hasattr(model, "module") else model + underlying._pipeline_mode = True + + # The pipeline calls backward()+optimizer.step() internally inside + # progress(), leaving no in-loop hook point for gradient clipping. Clip via + # a full-backward hook (fires after autograd populates dense grads, before + # the optimizer step) to preserve parity with the sequential path's + # clip_grad_norm_(model.parameters(), max_norm=1.0). + if grad_clip_norm and grad_clip_norm > 0: + + def _clip_grads(_m: torch.nn.Module, _gi: Any, _go: Any) -> None: + torch.nn.utils.clip_grad_norm_( + model.parameters(), max_norm=grad_clip_norm + ) + + model.register_full_backward_hook(_clip_grads) + + return TrainPipelineSparseDist( + model=model, + optimizer=optimizer, + device=device, + execute_all_batches=True, + custom_model_fwd=_PipelineModelWrapper(model), + ) + + +@gin.configurable +def train_eval_loop( + rank: int, + model: torch.nn.Module, + optimizer: Optimizer, + metric_logger: MetricsLogger, + device: torch.device, + num_epochs: int, + num_train_batches: Optional[int] = None, + num_eval_batches: Optional[int] = None, + train_dataloader: Optional[torch.utils.data.DataLoader] = None, + eval_dataloader: Optional[torch.utils.data.DataLoader] = None, + output_trace: bool = False, + metric_log_frequency: int = 1, + checkpoint_frequency: int = 100, + eval_frequency: int = 1, + start_train_batch_idx: int = 0, + start_eval_batch_idx: int = 0, + use_pipeline: bool = False, + # lr_scheduler: to-do: Add a scheduler +) -> None: + train_batch_idx: int = start_train_batch_idx + eval_batch_idx: int = start_eval_batch_idx + profiler = Profiler(rank) if output_trace else None + assert train_dataloader is not None and eval_dataloader is not None + + eval_data_iterator = iter(eval_dataloader) + train_data_iterator = iter(train_dataloader) + + # 3-stage TorchRec pipeline (overlaps embedding a2a with dense compute). + # When enabled, progress() owns H2D copy, sparse-dist, fwd/bwd and the + # optimizer step; grad clipping moves to a full-backward hook (see builder). + train_pipeline = ( + build_train_pipeline(model, optimizer, device) if use_pipeline else None + ) + + for epoch in range(num_epochs): + train_dataloader.sampler.set_epoch(epoch) # pyre-ignore [16] + while True: + model.train() + if train_pipeline is not None: + try: + ( + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + num_candidates, + ) = train_pipeline.progress(train_data_iterator) + except StopIteration: + train_data_iterator = iter(train_dataloader) + break + else: + try: + sample = next(train_data_iterator) + except StopIteration: + train_data_iterator = iter(train_dataloader) + break + optimizer.zero_grad() + sample.to(device) + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + # pyre-ignore + sum(aux_losses.values()).backward() + torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0) + optimizer.step() + num_candidates = sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0] + metric_logger.update( + mode="train", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=num_candidates, + ) + if train_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log( + mode="train", + additional_logs={ + "losses": aux_losses, + }, + ) + if train_batch_idx % checkpoint_frequency == 0 and train_batch_idx > 0: + save_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + rank=rank, + batch_idx=train_batch_idx, + ) + train_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if train_batch_idx % eval_frequency == 0: + model.eval() + eval_batch_idx: int = 0 + with torch.no_grad(): + while True: + try: + sample = next(eval_data_iterator) + except StopIteration: + eval_data_iterator = iter(eval_dataloader) + sample = next(eval_data_iterator) + sample.to(device) + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = ( + # In pipeline mode the model takes the batch as one + # arg (see _PipelineModelWrapper / DlrmHSTU.forward). + model.forward(sample) + if use_pipeline + else model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + ) + metric_logger.update( + mode="eval", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + eval_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if eval_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log(mode="eval") + if ( + num_eval_batches is not None + and eval_batch_idx >= num_eval_batches + ): + break + for k, v in metric_logger.compute(mode="eval").items(): + print(f"{k}: {v}") + model.train() + # `num_train_batches` cap: None or 0 = run the whole window. >0 caps + # batches per window (mostly the streaming-resume test driver uses + # this to keep test windows short). + if num_train_batches and train_batch_idx >= num_train_batches: + break + + +def select_in_window_checkpoint_reason( + *, + train_batch_idx: int, + global_step: int, + elapsed_since_last_save: float, + in_window_checkpoint_frequency: int, + checkpoint_step_frequency: int, + checkpoint_time_interval_s: float, +) -> Optional[str]: + """Decide which (if any) in-window checkpoint cadence fires this batch. + + Pure / distributed-agnostic so it can be unit-tested without a real run. + The caller computes `elapsed_since_last_save` (broadcast from rank 0 in the + streaming loop) so all ranks pass the same value and reach the same verdict. + + Precedence (at most one save per batch): per-window-local batch count > + monotonic global step > wall-clock interval. Returns the trigger reason + string, or None when no cadence fires. A cadence is disabled when its + frequency/interval is 0 / 0.0. + + Counter conventions match the loop: `train_batch_idx` is already + post-incremented (>=1 on the first batch), and `global_step` is guarded + >0 so step 0 doesn't trivially satisfy `% N == 0`. + """ + if ( + in_window_checkpoint_frequency > 0 + and train_batch_idx % in_window_checkpoint_frequency == 0 + ): + return "in_window_batch" + if ( + checkpoint_step_frequency > 0 + and global_step > 0 + and global_step % checkpoint_step_frequency == 0 + ): + return "global_step" + if ( + checkpoint_time_interval_s > 0 + and elapsed_since_last_save >= checkpoint_time_interval_s + ): + return "time_interval" + return None + + +def _validate_split_contract( + saved: Optional[Dict[str, Any]], + live: Dict[str, Any], + rank: int, +) -> None: + """Guarantee the train:eval split (and the inputs the resume skip-offset + depends on) are unchanged across a crash/resume. + + `saved` is the contract recovered from the checkpoint (None on cold start or + legacy pre-holdout checkpoints). Any mismatch is fatal: continuing would + either desync the mid-window skip (duplicate/skip batches) or reassign users + so that previously held-out eval users get trained (leakage). Set + ALLOW_SPLIT_MISMATCH=1 to override (e.g. intentionally resuming a legacy + checkpoint into a holdout run, accepting the risk). + """ + allow = os.environ.get("ALLOW_SPLIT_MISMATCH", "0") == "1" + if saved is None: + # Legacy / cold-start checkpoint with no recorded contract. Only a + # problem if this run actually holds users out (tsp < 1.0): we cannot + # prove the earlier run used the same split. + if live.get("train_split_percentage", 1.0) < 1.0 and not allow: + raise RuntimeError( + "Resuming a checkpoint with NO saved split contract into a " + f"user-holdout run (train_split_percentage=" + f"{live['train_split_percentage']}). The earlier run's split " + "cannot be verified, so held-out eval users may have been " + "trained. Set ALLOW_SPLIT_MISMATCH=1 to override." + ) + return + mismatches = { + k: (saved.get(k), live.get(k)) + for k in live + if saved.get(k) != live.get(k) + } + if mismatches: + msg = ( + "Split/resume contract mismatch between checkpoint and current run: " + + ", ".join( + f"{k}: checkpoint={s!r} current={c!r}" for k, (s, c) in mismatches.items() + ) + + ". Resuming would desync the skip offset and/or leak held-out " + "users into training." + ) + if allow: + if rank == 0: + logger.warning("%s ALLOW_SPLIT_MISMATCH=1 set — continuing anyway.", msg) + else: + raise RuntimeError(msg + " Set ALLOW_SPLIT_MISMATCH=1 to override.") + elif rank == 0: + logger.info("Split/resume contract verified against checkpoint: %s", live) + + +@gin.configurable +def streaming_train_eval_loop( + rank: int, + model: torch.nn.Module, + optimizer: Optimizer, + metric_logger: MetricsLogger, + device: torch.device, + num_train_ts: int, + hstu_config: DlrmHSTUConfig, + embedding_table_configs: Dict[str, EmbeddingConfig], + num_train_batches: Optional[int] = None, + num_eval_batches: Optional[int] = None, + output_trace: bool = False, + metric_log_frequency: int = 1, + # Interim eval logging cadence gate. 0 (default) = OFF: skip the per-batch + # compute_and_log DURING an eval pass. Those interim calls only log + # partial-holdout metrics (progress/debug); the value we keep -- the durable + # metrics.jsonl, early-stop, and MLPerf EVAL_ACCURACY -- is the end-of-pass + # compute() over the whole holdout, so OFF is numerically neutral for the + # recorded eval AUC. >0 restores the old interim logging. $EVAL_INTERIM_LOG. + eval_interim_metrics: int = 0, + # MLPerf `train_loss` event cadence, in global train steps, INDEPENDENT of + # metric_log_frequency (the console/TB cadence). 0 = fall back to + # metric_log_frequency (preserves prior coupled behavior). Wired to + # $MLPERF_TRAIN_LOSS_LOG_FREQ via gin. + mlperf_train_loss_log_frequency: int = 0, + checkpoint_frequency: int = 100, + start_ts: int = 0, + persistent_loader: bool = False, + eval_every_n_windows: int = 1, + # Data-fraction eval cadence (mutually exclusive with eval_every_n_windows). + # >0 = eval every this FRACTION of the run's total training data, converted + # once to a global train-step interval. 0.0 = OFF (use the per-window + # cadence). Wired to $EVAL_EVERY_DATA_PCT via gin. + eval_every_data_pct: float = 0.0, + # SKIP_EVAL: skip the FIRST N scheduled (periodic) eval passes to cut eval + # overhead early in the run (model far from the AUC target). 0 (default) = + # OFF (eval from the first scheduled point). Keyed off the same + # global_step / absolute-ts eval grid as the cadence, so the skip set is + # identical across resume; the end-of-run final eval is never skipped. + # Wired to $SKIP_EVAL via gin. + skip_eval: int = 0, + double_buffer: bool = False, + # --- fixed user-holdout eval set --- + # Window range the fixed eval set is drawn from. None -> default to + # original_end_ts (start_ts + num_train_ts), the window just past training. + eval_holdout_ts: Optional[int] = None, + eval_holdout_num_windows: int = 1, + # --- resume / mid-window-exact-once knobs --- + resume_train_ts: Optional[int] = None, + resume_batch_idx_in_window: int = WINDOW_COMPLETE, + # Split contract recovered from the checkpoint (None on cold start or + # legacy checkpoints). Validated below against the live split so a resumed + # run cannot silently train a different user-split (would leak). + resume_split_contract: Optional[Dict[str, Any]] = None, + # True iff no checkpoint was loaded (genuine fresh run). Distinguishes a + # cold start (safe to establish a new split) from a resume that merely lacks + # a contract (legacy/non-streaming checkpoint), which the guard must reject. + resume_cold_start: bool = False, + in_window_checkpoint_frequency: int = 0, + # --- global step / wall-clock checkpoint cadences --- + checkpoint_step_frequency: int = 0, + checkpoint_time_interval_s: float = 0.0, + # --- gradient clipping (streaming path, dense params). 0.0 = OFF, which + # preserves legacy streaming behavior. Wired to $GRAD_CLIP_NORM via gin. --- + grad_clip_norm: float = 0.0, + # --- LR warmup (streaming path). Linearly ramp LR 0 -> base over the first + # N GLOBAL train steps, then hold. Scales BOTH the dense optimizer and + # the in-backward fused sparse-embedding optimizer. 0 = OFF (byte- + # identical). Wired to $LR_WARMUP_STEPS via gin. --- + lr_warmup_steps: int = 0, + # --- LR warmup floor (streaming path). Absolute LR the ramp STARTS from at + # gstep 0 (per group, clamped to <= that group's base LR). The ramp then + # goes start -> base over `lr_warmup_steps`. 0.0 (default) reproduces the + # 0 -> base ramp exactly (byte-identical). A small nonzero floor turns the + # warmup into a "partial warmup" that reclaims the near-dead head of the + # ramp (trading a little seed-divergence damping for faster time-to-target). + # Only has any effect when lr_warmup_steps > 0. Wired to + # $LR_WARMUP_START_LR via gin. --- + lr_warmup_start_lr: float = 0.0, + # --- diagnostic: log per-batch unique/total embedding-id counts --- + streaming_diag_unique_emb: bool = False, + # --- test-only failure injection knob --- + die_at_step: int = -1, + # MLPerf logger (rank-0-gated); None disables all MLPerf event emission. + mlperf_logger: Optional[Any] = None, +) -> None: + """Streaming train+eval loop with per-window (and optionally mid-window) + checkpoints. + + Resume semantics (set by train_ranker after `load_dmp_checkpoint` returns): + - resume_train_ts=None: cold start; honor `start_ts` as-is. + - resume_train_ts=N, resume_batch_idx_in_window=WINDOW_COMPLETE(-1): + previous run finished window N cleanly. Start at N+1 from sample 0. + - resume_train_ts=N, resume_batch_idx_in_window=K (K>=0): previous run + crashed mid-window after K completed batches. Re-enter window N and + skip the first K batches of THIS rank's per-rank sample list (deterministic + slice since `window_indices(N)` is a pure function of the anchor_ts cache). + + Checkpoint cadences (all independent; any combination may be enabled): + - `checkpoint_frequency`: window-granularity. End-of-window save every + Nth train_ts (and always on the final window). Uses WINDOW_COMPLETE. + - `in_window_checkpoint_frequency`: per-window-local batch count. Fires + every N batches *within* a window (counter resets each window). + - `checkpoint_step_frequency`: global-step granularity. Fires whenever + the monotonic `metric_logger.global_step['train']` hits a multiple of + N — i.e. a true "every 1000 steps" trigger that spans windows and + survives resume (global_step is restored from the checkpoint). + - `checkpoint_time_interval_s`: wall-clock granularity. Fires when at + least this many seconds have elapsed since the last save (e.g. 3600 + for hourly). Rank 0 owns the clock and broadcasts the decision so all + ranks save together (avoids the collective barrier in + `save_dmp_checkpoint` deadlocking on a split decision). + + All in-window triggers (`in_window_checkpoint_frequency`, + `checkpoint_step_frequency`, `checkpoint_time_interval_s`) route through + `_save_mid_window`, which stamps `batch_idx_in_window=K` so a crash leaves + a resumable partial-window checkpoint. End-of-window saves + (`checkpoint_frequency`) always use the WINDOW_COMPLETE sentinel. 0 / 0.0 + disables a given cadence (the default for all three fine-grained ones). + + `die_at_step` is a test-only hook: when `metric_logger.global_step['train']` + reaches this value, the process exits with code 42 right after the in-window + save fires. Used by the failure-injection test to crash at a deterministic + boundary and then resume. + """ + # Exactly one eval cadence may be active. eval_every_n_windows defaults to 1 + # (eval every window), so enabling the data-fraction cadence REQUIRES + # explicitly disabling the per-window one (EVAL_EVERY_N_WINDOWS=0). Fail fast + # on a contradictory config rather than silently picking one. + if (eval_every_data_pct and eval_every_data_pct > 0) and eval_every_n_windows > 0: + raise ValueError( + "Conflicting eval cadences: eval_every_data_pct=" + f"{eval_every_data_pct} (>0) AND eval_every_n_windows=" + f"{eval_every_n_windows} (>0). They are mutually exclusive. To use " + "the data-fraction cadence set EVAL_EVERY_N_WINDOWS=0; to use the " + "per-window cadence set EVAL_EVERY_DATA_PCT=0." + ) + # MLPerf train_loss cadence: independent of metric_log_frequency. 0 (the + # env-binding default) falls back to metric_log_frequency so unset behavior + # matches the prior coupled implementation. + mlperf_loss_every = ( + mlperf_train_loss_log_frequency + if mlperf_train_loss_log_frequency and mlperf_train_loss_log_frequency > 0 + else metric_log_frequency + ) + profiler = Profiler(rank) if output_trace else None + # Normalize the per-window caps: <=0 (the env-binding default) means "no cap + # = consume the full window". The eval-break check below is `is not None and + # eval_batch_idx >= num_eval_batches`, so a literal 0 would (wrongly) break + # after the first batch — map it to None instead for the full-holdout eval. + if num_eval_batches is not None and num_eval_batches <= 0: + num_eval_batches = None + if num_train_batches is not None and num_train_batches <= 0: + num_train_batches = None + dataset_class, kwargs = get_dataset() + kwargs["embedding_config"] = embedding_table_configs + dataset = HammerToTorchDataset( + dataset=dataset_class(hstu_config=hstu_config, is_inference=False, **kwargs) + ) + # Persistent path: build ONE DataLoader + a stateful sampler whose indices + # are swapped per window, so workers fork once and are reused across all + # windows (eliminates the per-window dataloader respawn + first-batch + # warmup). The non-persistent path recreates a DataLoader per window. + window_sampler: Optional[StreamingWindowSampler] = None + persistent_dl: Optional[DataLoader] = None + world_size = ( + torch.distributed.get_world_size() + if torch.distributed.is_initialized() + else 1 + ) + if persistent_loader: + window_sampler = StreamingWindowSampler(rank=rank, world_size=world_size) + persistent_dl = make_persistent_streaming_dataloader( + dataset=dataset, sampler=window_sampler + ) + + # The fixed user-holdout eval is yambda-specific (needs window_indices + + # the split API). Other streaming datasets (synthetic) keep the legacy + # per-window eval. Detect support once. + supports_holdout = hasattr(dataset.dataset, "eval_holdout_indices") + + # Fixed eval-holdout window range. Captured from the REQUESTED (start_ts, + # num_train_ts) BEFORE the resume block mutates them, so it is identical on + # cold start and on every resume (the supervisor relaunches with the same + # START_TS / NUM_TRAIN_TS). Defaults to the window just past training. + requested_end_ts = start_ts + num_train_ts + # Eval-cadence anchor: the ORIGINAL requested start_ts, captured BEFORE the + # resume block rebases start_ts. `_should_eval` keys the every-N-windows + # cadence off the absolute window ts relative to THIS anchor, so the eval + # grid (e.g. 150,160,170,...) is identical on cold start and on every resume. + # (Keying off the per-call loop index instead would re-anchor the grid to + # whatever window a mid-run resume happens to restart from.) + eval_anchor_ts = start_ts + # None (Python default) or <0 (the env-binding default) both mean "use the + # window just past training", which is stable across resume. + eval_holdout_ts_resolved = ( + eval_holdout_ts + if (eval_holdout_ts is not None and eval_holdout_ts >= 0) + else requested_end_ts + ) + + # Data-fraction eval cadence: convert eval_every_data_pct into a global + # train-step interval ONCE, over the ORIGINAL requested window range + # [eval_anchor_ts, requested_end_ts). Keying the later trigger off + # `global_step % eval_interval_steps` (global_step is monotonic and + # checkpoint-restored) makes the eval grid identical on cold start and on + # every resume, exactly like checkpoint_step_frequency. 0 => disabled. + eval_interval_steps = 0 + if eval_every_data_pct and eval_every_data_pct > 0: + # Per-rank batch size: the persistent loader carries it directly; the + # per-window path uses the same gin %batch_size (env BATCH_SIZE, + # default 1024 — matches make_streaming_dataloader.batch_size). + bs = ( + persistent_dl.batch_size + if persistent_dl is not None + else int(os.environ.get("BATCH_SIZE", "1024")) + ) + if hasattr(dataset.dataset, "total_train_anchors"): + # total_train_anchors does a full-range gather over the mmap'd uid + # array for ~billions of positions + a uid hash. It is slow + # (minutes, single-threaded) AND, run on every rank independently, + # a large per-rank skew source: a fast rank finishes and races into + # the first embedding all-to-all while slow ranks are still hashing, + # desyncing the NCCL collective stream and hanging the job. The + # result is a pure function of the (identical) dataset + split, so + # compute it ONCE on rank 0 and broadcast the scalar; ranks 1..N + # skip the gather entirely (no 8x mmap/CPU contention, no skew). + if world_size > 1 and torch.distributed.is_initialized(): + _tta = ( + dataset.dataset.total_train_anchors( # pyre-ignore[16] + eval_anchor_ts, requested_end_ts - eval_anchor_ts + ) + if rank == 0 + else 0 + ) + _tta_t = torch.tensor([_tta], dtype=torch.int64, device=device) + torch.distributed.broadcast(_tta_t, src=0) + total_train_anchors = int(_tta_t.item()) + else: + total_train_anchors = dataset.dataset.total_train_anchors( # pyre-ignore[16] + eval_anchor_ts, requested_end_ts - eval_anchor_ts + ) + total_train_steps = total_train_anchors // max(1, bs * world_size) + eval_interval_steps = max( + 1, round(eval_every_data_pct * total_train_steps) + ) + if rank == 0: + logger.info( + "[data-pct-eval] eval_every_data_pct=%.6g -> " + "eval_interval_steps=%d (total_train_anchors=%d bs=%d " + "world_size=%d total_train_steps=%d over windows [%d, %d))", + eval_every_data_pct, + eval_interval_steps, + total_train_anchors, + bs, + world_size, + total_train_steps, + eval_anchor_ts, + requested_end_ts, + ) + elif rank == 0: + logger.warning( + "[data-pct-eval] dataset %s has no total_train_anchors(); " + "data-fraction eval is DISABLED (no per-window eval either, " + "since EVAL_EVERY_N_WINDOWS must be 0 to reach here) — only the " + "final eval will run.", + type(dataset.dataset).__name__, + ) + + # The split is an immutable run contract: a silent change across resume + # would both desync the mid-window skip offset AND turn held-out eval users + # into trained users (leakage). Build the live contract and validate the + # one recovered from the checkpoint against it; abort on any mismatch unless + # ALLOW_SPLIT_MISMATCH=1 is set (e.g. deliberately resuming a legacy run). + live_split_contract: Optional[Dict[str, Any]] = None + if supports_holdout: + live_split_contract = { + "train_split_percentage": dataset.dataset._train_split_percentage, # pyre-ignore[16] + "split_salt": dataset.dataset._split_salt, # pyre-ignore[16] + "eval_holdout_ts": eval_holdout_ts_resolved, + "eval_holdout_num_windows": eval_holdout_num_windows, + "batch_size": persistent_dl.batch_size if persistent_dl is not None else None, + "world_size": world_size, + } + # Only validate on an actual resume. On a genuine cold start there is no + # prior split to verify and establishing this run's split is always safe; + # validating there would wrongly reject every fresh holdout run. A resume + # that lacks a contract (legacy/non-streaming checkpoint) is NOT a cold + # start and is still validated (and rejected) below. + if not resume_cold_start: + _validate_split_contract(resume_split_contract, live_split_contract, rank) + + # Apply resume hint: advance start_ts past the last completed window, or + # re-enter the partial window with a per-rank skip on its first iter. + # Shrink num_train_ts by the same amount so the resumed run finishes at + # the same final timestamp (start_ts + num_train_ts) as a fresh run would + # — i.e. resumed and uninterrupted produce identical total work. + first_skip_samples = 0 + if resume_train_ts is not None: + original_end_ts = start_ts + num_train_ts + if resume_batch_idx_in_window == WINDOW_COMPLETE: + new_start = resume_train_ts + 1 + if rank == 0: + logger.info( + "Resuming from completed train_ts=%d → start_ts=%d " + "(num_train_ts %d → %d)", + resume_train_ts, new_start, + num_train_ts, max(0, original_end_ts - new_start), + ) + start_ts = new_start + else: + if rank == 0: + logger.info( + "Resuming mid-window at train_ts=%d batch_idx_in_window=%d " + "(skipping batches already trained)", + resume_train_ts, + resume_batch_idx_in_window, + ) + start_ts = resume_train_ts + # `batch_size` is per-rank from the persistent dataloader (set via + # gin `make_persistent_streaming_dataloader.batch_size`). The + # skip-samples-per-rank below maps "K batches done" → "K * bs + # samples in this rank's index list", since each batch draws bs + # samples from this rank's deterministic round-robin slice. + assert persistent_dl is not None, ( + "Mid-window resume requires persistent_loader=True" + ) + first_skip_samples = resume_batch_idx_in_window * persistent_dl.batch_size + num_train_ts = max(0, original_end_ts - start_ts) + if num_train_ts == 0 and rank == 0: + logger.info( + "Resume target already reached (end_ts=%d, start_ts=%d) — " + "no further training windows; skipping straight to final eval.", + original_end_ts, start_ts, + ) + + if rank == 0: + logger.info( + "[grad-clip] streaming path gradient clipping %s (max_norm=%.4g via $GRAD_CLIP_NORM)", + "ENABLED" if (grad_clip_norm and grad_clip_norm > 0) else "OFF", + grad_clip_norm, + ) + + def _window_iter(ts: int, skip_samples: int = 0): + # TRAIN-only iterator: both branches exclude held-out eval users via + # train_window_indices / set_ts(train_only=True). (Eval uses the fixed + # holdout set, never this helper.) + if persistent_loader: + assert window_sampler is not None and persistent_dl is not None + window_sampler.set_window( + dataset.dataset.train_window_indices(ts), # pyre-ignore [16] + skip_samples=skip_samples, + ) + return iter(persistent_dl) + if skip_samples != 0: + raise NotImplementedError( + "skip_samples>0 requires persistent_loader=True" + ) + return iter( + make_streaming_dataloader(dataset=dataset, ts=ts, train_only=True) + ) + # Windows are [start_ts, start_ts + num_train_ts); each step trains window T + # then evals window T+1, so the last eval window is start_ts + num_train_ts, + # which must be < num_windows(). Anchors require >= history_length prior + # events, so the earliest windows are near-empty warm-up — use start_ts to + # begin at a dense window. Clamp instead of failing. + if hasattr(dataset.dataset, "num_windows"): + available = dataset.dataset.num_windows() # pyre-ignore [16] + max_count = max(0, available - 1 - start_ts) + if num_train_ts > max_count: + logger.warning( + f"start_ts={start_ts} + num_train_ts={num_train_ts} exceeds " + f"available windows ({available}); clamping num_train_ts to {max_count}." + ) + num_train_ts = max_count + # Wall-clock anchor for time-based checkpointing. Mutable single-element + # list so the nested train loop can reset it after each save. Starts at + # loop entry so the first time-trigger fires ~interval seconds in. + last_ckpt_time = [time.time()] + + def _broadcast_elapsed() -> float: + """Seconds since the last save, owned by rank 0 and broadcast to all + ranks. save_dmp_checkpoint runs a collective barrier, so every rank must + feed the same wall-clock value into the cadence decision — otherwise a + split verdict (rank 0 saves, rank 1 doesn't) would deadlock. Broadcasting + rank 0's elapsed keeps the (pure) decision identical everywhere.""" + elapsed = time.time() - last_ckpt_time[0] + if torch.distributed.is_initialized() and world_size > 1: + t = torch.tensor([elapsed], device=device, dtype=torch.float64) + torch.distributed.broadcast(t, src=0) + elapsed = float(t.item()) + return elapsed + + def _save_mid_window(train_ts: int, batch_idx_in_window: int) -> None: + """In-window checkpoint helper. Snapshots the same state as the + end-of-window save but stamps `batch_idx_in_window=K` instead of + WINDOW_COMPLETE so the resume path knows to skip K batches. + Uses train_ts as the numeric subdir name — every save into the same + train_ts overwrites the previous in-window snapshot (via atomic + replace), so disk stays bounded to keep_last_n train_ts dirs.""" + save_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + rank=rank, + batch_idx=train_ts, + train_ts=train_ts, + batch_idx_in_window=batch_idx_in_window, + device=device, + split_contract=live_split_contract, + ) + + # --- LR warmup (streaming path) ------------------------------------------ + # Linearly ramp LR from 0 -> base over the first `lr_warmup_steps` GLOBAL + # train steps, then hold at base. Keyed off the monotonic, checkpoint- + # restored global_step so the ramp is identical on cold start and on every + # resume (a run that crashes mid-warmup resumes at the correct multiplier; + # a run already past warmup is a no-op). Scales BOTH LR channels: the + # dense/non-fused optimizer passed in as `optimizer`, AND the in-backward + # fused sparse-embedding optimizer (model.fused_optimizer), whose per-table + # LR dominates this model. `lr_warmup_steps=0` (default) => OFF and the base + # LRs are never touched (byte-identical to the pre-warmup path). + _warmup_groups = list(optimizer.param_groups) + _warmup_fused_opt = getattr(model, "fused_optimizer", None) + if _warmup_fused_opt is not None: + _warmup_groups += list(_warmup_fused_opt.param_groups) + _warmup_base_lrs = [pg["lr"] for pg in _warmup_groups] + # Per-group start LR for the ramp: the configured floor, clamped so it can + # never exceed a group's base (a floor >= base would mean "no ramp" for that + # group, which we express by starting it AT base). 0.0 => classic 0 -> base. + _warmup_start_lrs = [min(float(lr_warmup_start_lr), b) for b in _warmup_base_lrs] + if lr_warmup_steps and lr_warmup_steps > 0 and rank == 0: + logger.info( + "[lr-warmup] linear warmup over %d global steps across %d param " + "group(s); start LRs: %s -> base LRs: %s", + lr_warmup_steps, + len(_warmup_groups), + ", ".join(f"{s:.3e}" for s in _warmup_start_lrs), + ", ".join(f"{b:.3e}" for b in _warmup_base_lrs), + ) + + def _apply_lr_warmup(gstep: int) -> None: + # OFF, or ramp already finished (kernel + param_groups already hold the + # base LR from the last sync at gstep==lr_warmup_steps): nothing to do. + if not (lr_warmup_steps and lr_warmup_steps > 0) or gstep > lr_warmup_steps: + return + progress = min(1.0, float(gstep) / float(lr_warmup_steps)) + # Ramp each group from its (clamped) start LR up to its base LR. With + # start=0 this collapses to `base * progress` (the original schedule). + for pg, base, start in zip( + _warmup_groups, _warmup_base_lrs, _warmup_start_lrs + ): + pg["lr"] = start + (base - start) * progress + # Push the warmed LR into the FBGEMM TBE backward kernel. The in-backward + # fused embedding optimizer only calls + # `emb_module.set_learning_rate(param_groups[0]["lr"])` from its + # step()/zero_grad() (torchrec/distributed/batched_embedding_kernel.py), + # and this loop steps the dense `optimizer` — NOT model.fused_optimizer — + # so WITHOUT this explicit call the embedding tables would keep training + # at the construction-time base LR and warmup would silently apply to the + # dense params only. `.step()` on the fused (in-backward) optimizer does + # no parameter update; it only re-syncs the LR to the kernel. + if _warmup_fused_opt is not None: + _warmup_fused_opt.step() + if rank == 0 and gstep % max(1, metric_log_frequency) == 0: + _fused_lr = "" + if _warmup_fused_opt is not None: + _fused_lr = ( + f" fused_pg_lr={_warmup_fused_opt.param_groups[0]['lr']:.3e}" + ) + # Best-effort readback of the ACTUAL kernel LR to confirm + # set_learning_rate() propagated. model.fused_optimizer is a + # (possibly nested) CombinedOptimizer whose leaves carry the TBE + # module as `_emb_module`, so recurse to the first leaf. Purely + # diagnostic; wrapped so it can never break training. + def _first_tbe_kernel_lr(opt, depth=0): + if depth > 6 or opt is None: + return None + _em = getattr(opt, "_emb_module", None) + if _em is not None and hasattr(_em, "get_learning_rate"): + return float(_em.get_learning_rate()) + for _sub in getattr(opt, "_optims", []): + _child = _sub[1] if isinstance(_sub, tuple) else _sub + _lr = _first_tbe_kernel_lr(_child, depth + 1) + if _lr is not None: + return _lr + _wrapped = getattr(opt, "_optimizer", None) + if _wrapped is not None: + return _first_tbe_kernel_lr(_wrapped, depth + 1) + return None + + try: + _klr = _first_tbe_kernel_lr(_warmup_fused_opt) + if _klr is not None: + _fused_lr += f" tbe_kernel_lr={_klr:.3e}" + except Exception: + pass + logger.info( + "[lr-warmup] gstep=%d progress=%.4f dense_lr=%.3e%s", + gstep, + progress, + _warmup_groups[0]["lr"], + _fused_lr, + ) + + def _run_train_window( + train_data_iterator, + train_ts: int, + start_batch_idx: int = 0, + label: Optional[str] = None, + do_eval: Optional[Callable[[int, int], None]] = None, + ) -> None: + # `start_batch_idx` is set when we're re-entering a window that was + # interrupted mid-way (in_window resume); the dataloader iterator was + # already advanced past those batches via the sampler skip, and we + # account for them in the local counter so in-window saves and the + # die_at_step hook fire at the right relative offsets. + train_batch_idx = start_batch_idx + first_wait: Optional[float] = None + while True: + model.train() + _t_next = time.perf_counter() if (label and rank == 0) else None + try: + sample = next(train_data_iterator) + except StopIteration: + break + if _t_next is not None and first_wait is None: + first_wait = time.perf_counter() - _t_next + if streaming_diag_unique_emb and train_batch_idx < int( + os.environ.get("DIAG_EMB_STEPS", "100") + ): + _log_unique_embedding_diag( + sample, + rank, + train_batch_idx, + max_steps=int(os.environ.get("DIAG_EMB_STEPS", "100")), + log_every=metric_log_frequency, + ) + # LR warmup: set the scaled LR on BOTH channels (dense param_groups + # + the fused embedding TBE via set_learning_rate) BEFORE the + # forward/backward, so the in-backward embedding update THIS step + # uses the warmed LR (the dense optimizer reads it at .step() below). + # No-op when lr_warmup_steps=0. + _apply_lr_warmup(metric_logger.global_step["train"]) + optimizer.zero_grad() + sample.to(device) + ( + _, + _, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + # pyre-ignore + sum(aux_losses.values()).backward() + # Gradient clipping for the streaming path. Clips dense params (the + # sparse embedding tables use a fused optimizer and are unaffected, + # same as the non-streaming path's clip_grad_norm_). OFF by default + # (grad_clip_norm=0.0 via $GRAD_CLIP_NORM) so legacy streaming runs + # are byte-for-byte unchanged; set >0 to enable. + if grad_clip_norm and grad_clip_norm > 0: + torch.nn.utils.clip_grad_norm_( + model.parameters(), max_norm=grad_clip_norm + ) + optimizer.step() + metric_logger.update( + mode="train", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + # MLPerf train_loss event on its own cadence (decoupled from the + # console/TB metric cadence below). Called every step; the cross-rank + # all-reduce only fires on the cadence, gated by the rank-identical + # global_step inside the method, so it stays in lockstep. + metric_logger.maybe_log_mlperf_train_loss(aux_losses, every=mlperf_loss_every) + if train_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log( + mode="train", + additional_logs={ + "losses": aux_losses, + }, + ) + train_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + # Fine-grained in-window checkpoint triggers. All stamp + # batch_idx_in_window so a crash here leaves a resumable partial + # checkpoint, and all fire AFTER the metric update so restored + # state reflects the just-completed batch. Triggers are mutually + # short-circuited (one save per batch max) but evaluated on the + # same deterministic counters across all ranks, so the collective + # inside save_dmp_checkpoint stays in lockstep. + gstep = metric_logger.global_step["train"] + # Wall-clock elapsed is broadcast from rank 0 so every rank feeds + # the same value into the (otherwise pure) cadence decision. + elapsed = ( + _broadcast_elapsed() if checkpoint_time_interval_s > 0 else 0.0 + ) + save_reason = select_in_window_checkpoint_reason( + train_batch_idx=train_batch_idx, + global_step=gstep, + elapsed_since_last_save=elapsed, + in_window_checkpoint_frequency=in_window_checkpoint_frequency, + checkpoint_step_frequency=checkpoint_step_frequency, + checkpoint_time_interval_s=checkpoint_time_interval_s, + ) + if save_reason is not None: + if rank == 0: + logger.info( + "checkpoint trigger=%s train_ts=%d batch=%d global_step=%d", + save_reason, + train_ts, + train_batch_idx, + gstep, + ) + _save_mid_window(train_ts, train_batch_idx) + # Reset the wall-clock anchor on ANY save so the next time + # trigger is measured from the most recent checkpoint. + last_ckpt_time[0] = time.time() + # Data-fraction eval cadence: run the full-holdout eval whenever the + # monotonic global step crosses a multiple of eval_interval_steps + # (i.e. every eval_every_data_pct of the training data). Keyed off + # global_step (checkpoint-restored) so the eval grid is identical + # across resume. Mid-window-safe: eval sets model.eval(), so restore + # train mode + dataset.is_eval afterward. do_eval is None unless the + # data-pct cadence is enabled. + if ( + do_eval is not None + and eval_interval_steps > 0 + and gstep > 0 + and gstep % eval_interval_steps == 0 + ): + # SKIP_EVAL: the 1-based ordinal of this data-pct eval boundary is + # gstep / eval_interval_steps. Skip it (keep training) while that + # ordinal is within the first `skip_eval` points. Ordinal is a pure + # function of the checkpoint-restored global_step, so the skip set + # is identical across resume. + _eval_ordinal = gstep // eval_interval_steps + if skip_eval > 0 and _eval_ordinal <= skip_eval: + if rank == 0: + logger.info( + "[data-pct-eval] SKIP eval train_ts=%d global_step=%d " + "ordinal=%d <= SKIP_EVAL=%d", + train_ts, + gstep, + _eval_ordinal, + skip_eval, + ) + else: + if rank == 0: + logger.info( + "[data-pct-eval] trigger eval train_ts=%d global_step=%d " + "(interval=%d)", + train_ts, + gstep, + eval_interval_steps, + ) + do_eval(train_ts, gstep) + model.train() + dataset.dataset.is_eval = False # pyre-ignore [16] + # Data-fraction eval may hit the MLPerf target and emit RUN_STOP + # (via _do_eval_*). Stop the window immediately so we don't train + # past the convergence point; the outer window loop checks the + # same flag and breaks too. + if mlt.run_stopped: + break + # Test-only: deterministic crash for the failure-injection test. + # Triggered AFTER the save above, so on resume we re-enter at + # batch_idx_in_window=train_batch_idx and emit batches [K+1, end). + if ( + die_at_step >= 0 + and metric_logger.global_step["train"] >= die_at_step + ): + if rank == 0: + logger.warning( + "die_at_step=%d hit at train_ts=%d batch=%d global_step=%d " + "→ sys.exit(42)", + die_at_step, + train_ts, + train_batch_idx, + metric_logger.global_step["train"], + ) + # Distributed barrier so all ranks exit together rather than + # leaving a few ranks hanging on NCCL ops. + torch.distributed.barrier() + import sys + sys.exit(42) + # `num_train_batches` cap: None or 0 = run the whole window. >0 caps + # batches per window (mostly the streaming-resume test driver uses + # this to keep test windows short). + if num_train_batches and train_batch_idx >= num_train_batches: + break + if label and rank == 0 and first_wait is not None: + logger.info( + f"[boundary] {label} train first-batch data-wait={first_wait * 1000:.1f}ms" + ) + + def _run_eval_window( + eval_data_iterator, label: Optional[str] = None + ) -> Dict[str, float]: + # DO NOT add a checkpoint trigger anywhere inside this function. The eval + # data iterator's position is not serializable, so a checkpoint taken + # mid-eval could not be resumed deterministically. `_maybe_checkpoint` + # only fires after a completed eval window or mid-train-window, so any + # restored state always sits on a completed-eval boundary -- which is + # also why the eval reset below is safe across resume. + # + # Exclude this eval pass's wall-time from the train step-time window so + # step_ms stays canonical even when eval coincides with a train interval; + # the duration is reported separately (window_eval_time_ms + total_eval + # below). Resumed unconditionally at the end of this function. + metric_logger.pause_perf("eval") + model.eval() + # Reset eval metrics so each pass reports a clean number over the FIXED + # holdout set. Without this, lifetime/window eval metrics would keep + # accumulating across eval steps (the old behavior, made worse now that + # every step sees the identical set), making the eval-AUC trajectory + # uninterpretable. With the reset, each eval point == AUC over the whole + # fixed holdout at that train step -> directly comparable across steps. + metric_logger.reset(mode="eval") + eval_batch_idx = 0 + first_wait: Optional[float] = None + _t_enter = time.perf_counter() if (label and rank == 0) else None + with torch.no_grad(): + while True: + _t_next = time.perf_counter() if (label and rank == 0) else None + try: + sample = next(eval_data_iterator) + except StopIteration: + break + if _t_next is not None and first_wait is None: + first_wait = time.perf_counter() - _t_next + sample.to(device) + ( + _, + _, + _, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) = model.forward( + sample.uih_features_kjt, + sample.candidates_features_kjt, + ) + metric_logger.update( + mode="eval", + predictions=mt_target_preds, + labels=mt_target_labels, + weights=mt_target_weights, + num_candidates=sample.candidates_features_kjt.lengths().view( + len(sample.candidates_features_kjt.keys()), -1 + )[0], + ) + eval_batch_idx += 1 + if output_trace: + assert profiler is not None + profiler.step() + if eval_interim_metrics and eval_batch_idx % metric_log_frequency == 0: + metric_logger.compute_and_log(mode="eval") + if num_eval_batches is not None and eval_batch_idx >= num_eval_batches: + break + _eval_metrics = metric_logger.compute(mode="eval") + for k, v in _eval_metrics.items(): + print(f"{k}: {v}") + if label and rank == 0 and _t_enter is not None: + _eval_total = time.perf_counter() - _t_enter + _fw = (first_wait * 1000) if first_wait is not None else float("nan") + logger.info( + f"[boundary] {label} eval first-batch data-wait={_fw:.1f}ms " + f"total_eval={_eval_total * 1000:.1f}ms batches={eval_batch_idx}" + ) + # Dedicated per-eval metrics sink. One JSON line per eval boundary + # capturing the END-OF-PASS metric over the FIXED holdout -- the single + # correct value for that eval point (no interim/averaging ambiguity). + # Rank 0 only; append-only so it survives restarts and the trajectory + # accumulates across resumes. Written next to the main run log + # (".metrics.jsonl"), falling back to cwd if LOG is unset. + import json + import re as _re + + _log = os.environ.get("LOG") + if _log: + _base = _log[:-4] if _log.endswith(".log") else _log + _metrics_path = f"{_base}.metrics.jsonl" + else: + _metrics_path = "streaming_eval_metrics.jsonl" + _ts_m = _re.search(r"train_ts=(\d+)", label) + _rec = { + "label": label, + "train_ts": int(_ts_m.group(1)) if _ts_m else None, + "global_step": int(metric_logger.global_step.get("train", -1)), + "eval_batches": eval_batch_idx, + "total_eval_ms": round(_eval_total * 1000, 1), + "wall_time": time.time(), + } + for _k, _v in _eval_metrics.items(): + try: + _rec[_k] = float(_v) + except (TypeError, ValueError): + pass + try: + with open(_metrics_path, "a") as _f: + _f.write(json.dumps(_rec) + "\n") + except OSError as _e: + logger.warning("failed to write metrics sink %s: %s", _metrics_path, _e) + metric_logger.resume_perf("eval") + # Return metrics (on every rank) so the MLPerf eval hooks can consume them. + return _eval_metrics + + def _maybe_checkpoint(train_ts: int) -> None: + if ( + train_ts % checkpoint_frequency == 0 and train_ts > 0 + ) or train_ts == start_ts + num_train_ts - 1: + # End-of-window save: stamp WINDOW_COMPLETE so resume advances past + # this train_ts. `device` enables per-rank RNG snapshot for + # bit-equal resume of dropout-bearing modules. + save_dmp_checkpoint( + model=model, + optimizer=optimizer, + metric_logger=metric_logger, + rank=rank, + batch_idx=train_ts, + train_ts=train_ts, + batch_idx_in_window=WINDOW_COMPLETE, + device=device, + split_contract=live_split_contract, + ) + last_ckpt_time[0] = time.time() + + # Apply start_ts shift from resume (may have moved past the original start). + # num_train_ts is the requested *count*; preserve it so the loop runs for + # the same total number of windows post-resume as a fresh run would have. + train_ts_list = list(range(start_ts, start_ts + num_train_ts)) + n_train = len(train_ts_list) + + def _should_eval(i: int) -> bool: + """Whether to run the full-holdout eval after training window index `i`. + + Single cadence knob `eval_every_n_windows`: + * <=0 -> eval disabled entirely (train-only; e.g. perf benchmarking or + the resume test). The eval dataloader is not even built. + * 1 (default) -> eval after every window. + * K>1 -> eval when the ABSOLUTE window ts is on the grid anchored at + `eval_anchor_ts` (the original start_ts), i.e. ts in {anchor, + anchor+K, anchor+2K, ...}, and ALWAYS on the final window so the + trajectory ends with an eval point. Anchoring to the absolute ts + (not the per-call loop index `i`) keeps the eval grid (e.g. + 150,160,170,...) stable across a mid-run resume, which rebases + start_ts/`train_ts_list` to the resume window. + """ + if eval_every_n_windows <= 0: + return False + if eval_every_n_windows == 1: + return True + return (train_ts_list[i] - eval_anchor_ts) % eval_every_n_windows == 0 or i == n_train - 1 + + def _skip_early_window_eval(i: int) -> bool: + """SKIP_EVAL for the PER-WINDOW cadence: skip the first `skip_eval` + eval-eligible points on the absolute-ts grid so early (low-signal) eval + passes are not run. The ordinal is derived from the absolute ts + ((train_ts - eval_anchor_ts) / eval_every_n_windows), so the skip set is + stable across a mid-run resume. The final window's eval is never skipped + so the trajectory always ends with an eval point. + """ + if skip_eval <= 0 or i == n_train - 1: + return False + ordinal = (train_ts_list[i] - eval_anchor_ts) // max(1, eval_every_n_windows) + return ordinal < skip_eval + + # Fixed eval set: held-out users' anchors over the resolved holdout window + # range, computed ONCE and reused at every eval step. Same anchors every + # step -> stable, comparable eval-AUC curve, and bounded eval time + # (~(1 - train_split_percentage) of a window). Cached inside the dataset so + # re-deriving it (e.g. on resume) returns the identical set. None for + # datasets without holdout support (synthetic) -> legacy per-window eval. + eval_global_indices: Optional["np.ndarray"] = None + if supports_holdout: + eval_global_indices = dataset.dataset.eval_holdout_indices( # pyre-ignore [16] + eval_holdout_ts_resolved, eval_holdout_num_windows + ) + if rank == 0: + logger.info( + "Fixed eval holdout: ts=[%d, %d) -> %d anchors (train_split_percentage=%s)", + eval_holdout_ts_resolved, + eval_holdout_ts_resolved + eval_holdout_num_windows, + len(eval_global_indices), + dataset.dataset._train_split_percentage, # pyre-ignore[16] + ) + + # --- MLPerf run tracking -------------------------------------------------- + # total_train_samples = epoch_num denominator (global trainable samples over + # the window range), computed once and logged as TRAIN_SAMPLES. + total_train_samples = 0 + if mlperf_logger is not None: + _idx_fn = getattr( + dataset.dataset, "train_window_indices", None + ) or getattr(dataset.dataset, "window_indices", None) + if _idx_fn is not None: + for _ts in train_ts_list: + total_train_samples += int(_idx_fn(_ts).size) + if rank == 0: + logger.info( + "MLPerf: total_train_samples=%d over %d windows", + total_train_samples, + n_train, + ) + # Wire the logger + LR getter so MetricsLogger.compute emits train_loss. + metric_logger.mlperf_logger = mlperf_logger + + def _current_lr() -> float: + return float(optimizer.param_groups[0]["lr"]) + + metric_logger.lr_getter = _current_lr + + # Centralized MLPerf run-boundary state machine: owns block/eval/run markers, + # SAMPLES_COUNT/EPOCH_NUM progress metadata, and the per-window-AUC vs + # auc_threshold convergence decision. Every method no-ops when mlperf_logger + # is None, so the loop below calls them unconditionally. + from generative_recommenders.dlrm_v3.train.mlperf_logging_utils import ( + MLPerfRunTracker, + ) + + mlt = MLPerfRunTracker( + logger=mlperf_logger, + metric_logger=metric_logger, + total_train_samples=total_train_samples, + rank=rank, + device=device, + ) + mlt.log_dataset_sizes( + eval_samples=eval_global_indices.size + if eval_global_indices is not None + else None + ) + + if persistent_loader and double_buffer: + # Double-buffered: next window prepared in the background during the + # current window's compute. Eval (if enabled) uses its own pre-forked + # pool, primed up front on the main thread so no fork races a bg thread. + prefetcher = _PrefetchingWindowLoader( + dataset=dataset, + sampler_factory=lambda: StreamingWindowSampler(rank, world_size), + dl_factory=lambda s: make_persistent_streaming_dataloader( + dataset=dataset, sampler=s + ), + ) + eval_sampler: Optional[StreamingWindowSampler] = None + eval_dl: Optional[DataLoader] = None + # Eval iterator is built one window ahead: the eval pool (idle while the + # current train window runs) prefetches the next eval window's first + # batches concurrently with train compute, so eval starts warm. yambda's + # sample content depends only on the sampler window, not is_eval, so + # prefetching during train is safe. + eval_iter: Optional[Iterator] = None + # Build/fork the eval pool when EITHER cadence needs it: the per-window + # cadence (eval_every_n_windows>0) or the data-fraction cadence + # (eval_interval_steps>0). Both are never simultaneously on (validated + # at entry), so this is "eval is enabled at all". + if (eval_every_n_windows > 0 or eval_interval_steps > 0) and len( + train_ts_list + ) > 0: + eval_sampler = StreamingWindowSampler(rank, world_size) + eval_dl = make_persistent_streaming_dataloader( + dataset=dataset, sampler=eval_sampler + ) + # CRITICAL: fork the eval worker pool HERE, on the main thread, + # BEFORE prefetcher.stream() below spins up its background prep + # thread. The pool is persistent_workers=True, so this first iter() + # is the ONLY fork; every later iter() merely resets and reuses these + # workers (no fork), so it can never deadlock against the background + # thread holding an allocator/GIL-released lock. (Deferring this + # first fork into the loop — as a sparse-eval cadence naively might — + # hangs the run.) _should_eval(0) is always True when eval is enabled + # (0 % K == 0). The eval set is the FIXED holdout (same every step), + # so we install it on the sampler ONCE here; later evals just call + # iter() again to replay the identical set (no set_window churn). + eval_sampler.set_window(eval_global_indices) + eval_iter = iter(eval_dl) + + # Data-fraction eval callback (double-buffer path). Fired mid-window by + # _run_train_window on the global-step cadence. Reuses the already-forked + # persistent eval pool: iter(eval_dl) here runs on the MAIN thread (a + # reset, not a fork — the only fork was the up-front iter() above), so it + # stays safe alongside the background window-prefetch thread. + def _do_eval_db(train_ts: int, gstep: int) -> None: + # Data-fraction eval boundary: this closes the current MLPerf block, + # runs the holdout eval with full EVAL_START/EVAL_STOP + EVAL_ACCURACY + # + convergence, then opens the next block. The block thus brackets + # exactly one eval_interval_steps of training (MLPerf block == work + # between two evals), instead of one timestamp window. + dataset.dataset.is_eval = True # pyre-ignore [16] + assert eval_dl is not None + mlt.block_stop() + mlt.eval_start() + eval_metrics = _run_eval_window( + iter(eval_dl), + label=f"eval_holdout@train_ts={train_ts}@step={gstep}", + ) + # Emits RUN_STOP (sets mlt.run_stopped) if the target is met; + # _run_train_window / the window loop break on that flag. + mlt.eval_stop(eval_metrics) + if not mlt.run_stopped: + mlt.block_start() + + _db_do_eval = _do_eval_db if eval_interval_steps > 0 else None + + # Block placement depends on the eval cadence. Per-window cadence + # (eval_every_n_windows>0): one block per timestamp window. Otherwise + # (data-fraction cadence, or no eval): a single block spans the whole + # run, split at each data-fraction eval boundary by _do_eval_db. Open + # the first block here for the latter; the boundary helper + the + # post-loop stop handle the rest. + _per_window_blocks = eval_every_n_windows > 0 + if not _per_window_blocks: + mlt.block_start() + + for i, (train_ts, train_data_iterator) in enumerate( + # Only the FIRST window after a mid-window resume needs the skip + # (handed via prefetcher.stream's first_skip_samples). The skip is + # zero on cold start (resume_train_ts is None → first_skip_samples=0) + # and on completed-window resume (mid-window slice is 0 too). + prefetcher.stream(train_ts_list, first_skip_samples=first_skip_samples) + ): + dataset.dataset.is_eval = False # pyre-ignore [16] + # First iteration after a mid-window resume carries + # resume_batch_idx_in_window so in-window saves and the die_at_step + # hook keep accurate counters; otherwise count from 0. + start_batch = ( + resume_batch_idx_in_window + if i == 0 and resume_batch_idx_in_window > 0 + else 0 + ) + # Rendezvous all ranks at the window boundary BEFORE the first + # forward of this window. The prefetcher has already handed back a + # ready iterator (this window's window_indices mmap scan is done), + # but that O(N) scan over the ~18 GB anchor_ts array can finish at + # very different times across ranks. Without this barrier a fast + # rank issues the first embedding all-to-all while a slow rank is + # still in prep, desyncing the NCCL collective stream and hanging + # the job (observed at a window boundary via the flight recorder: + # ranks split across consecutive collective seq ids). This only + # absorbs prep skew (one near-zero sync per window); it does not + # serialize the background prefetch of future windows. + _window_boundary_barrier(device, world_size, train_ts) + if _per_window_blocks: + mlt.block_start() + _run_train_window( + train_data_iterator, + train_ts=train_ts, + start_batch_idx=start_batch, + label=f"train_ts={train_ts}", + do_eval=_db_do_eval, + ) + if _per_window_blocks: + mlt.block_stop() + should_stop = False + if _should_eval(i) and not _skip_early_window_eval(i): + dataset.dataset.is_eval = True # pyre-ignore [16] + assert eval_sampler is not None and eval_dl is not None + mlt.eval_start() + eval_metrics = _run_eval_window( + eval_iter, label=f"eval_holdout@train_ts={train_ts}" + ) + should_stop = mlt.eval_stop(eval_metrics) + # Re-arm the (already-forked) eval pool for the NEXT eval. The + # holdout set is fixed, so the sampler window is unchanged; we + # only need a fresh iter() to replay it. iter() reuses the + # persistent workers — no fork, safe alongside the bg thread. + next_eval_i = next( + (j for j in range(i + 1, n_train) if _should_eval(j)), None + ) + if next_eval_i is not None: + eval_iter = iter(eval_dl) + _maybe_checkpoint(train_ts) + # should_stop: per-window convergence. mlt.run_stopped: + # data-fraction convergence (RUN_STOP fired mid-window by _do_eval_db). + if should_stop or mlt.run_stopped: + # MLPerf target reached: RUN_STOP already emitted; stop training. + break + + # Close the run-spanning block for the data-fraction / no-eval case. + # Idempotent: a no-op if the last eval boundary already closed it (i.e. + # convergence stopped the run) or if per-window blocks were used. + if not _per_window_blocks: + mlt.block_stop() + else: + # Data-fraction eval callback (non-double-buffer path). Builds a fresh + # eval dataloader per call over the FIXED holdout set (or the legacy + # next-window eval when the dataset has no holdout support). + def _do_eval_nb(train_ts: int, gstep: int) -> None: + # Data-fraction eval boundary (non-double-buffer path). See _do_eval_db: + # close the current MLPerf block, run the eval with full markers + + # convergence, then open the next block so a block brackets one + # eval_interval_steps of training rather than a timestamp window. + dataset.dataset.is_eval = True # pyre-ignore [16] + mlt.block_stop() + mlt.eval_start() + if eval_global_indices is not None: + eval_metrics = _run_eval_window( + iter( + make_streaming_dataloader( + dataset=dataset, indices=eval_global_indices + ) + ), + label=f"eval_holdout@train_ts={train_ts}@step={gstep}", + ) + else: + eval_metrics = _run_eval_window( + iter(make_streaming_dataloader(dataset=dataset, ts=train_ts + 1)), + label=f"eval@train_ts={train_ts}@step={gstep}", + ) + mlt.eval_stop(eval_metrics) + if not mlt.run_stopped: + mlt.block_start() + + _nb_do_eval = _do_eval_nb if eval_interval_steps > 0 else None + + # See the double-buffer branch: per-window blocks for the per-window + # cadence, else a single run-spanning block split at data-fraction eval + # boundaries by _do_eval_nb. + _per_window_blocks = eval_every_n_windows > 0 + if not _per_window_blocks: + mlt.block_start() + + for i, train_ts in enumerate(train_ts_list): + dataset.dataset.is_eval = False # pyre-ignore [16] + skip = first_skip_samples if i == 0 else 0 + start_batch = ( + resume_batch_idx_in_window + if i == 0 and resume_batch_idx_in_window > 0 + else 0 + ) + # See the double-buffer path: rendezvous all ranks at the window + # boundary before the first forward so per-rank data-prep skew + # cannot desync the NCCL collective stream and hang the job. + _window_boundary_barrier(device, world_size, train_ts) + if _per_window_blocks: + mlt.block_start() + _run_train_window( + _window_iter(train_ts, skip_samples=skip), + train_ts=train_ts, + start_batch_idx=start_batch, + do_eval=_nb_do_eval, + ) + if _per_window_blocks: + mlt.block_stop() + should_stop = False + if _should_eval(i) and not _skip_early_window_eval(i): + dataset.dataset.is_eval = True # pyre-ignore [16] + mlt.eval_start() + if eval_global_indices is not None: + eval_metrics = _run_eval_window( + iter( + make_streaming_dataloader( + dataset=dataset, indices=eval_global_indices + ) + ), + label=f"eval_holdout@train_ts={train_ts}", + ) + else: + # Legacy per-window eval (datasets without user holdout). + eval_metrics = _run_eval_window( + iter(make_streaming_dataloader(dataset=dataset, ts=train_ts + 1)) + ) + should_stop = mlt.eval_stop(eval_metrics) + elif _should_eval(i) and rank == 0: + logger.info( + "[per-window-eval] SKIP eval train_ts=%d (SKIP_EVAL=%d)", + train_ts, + skip_eval, + ) + _maybe_checkpoint(train_ts) + # should_stop: per-window convergence. mlt.run_stopped: + # data-fraction convergence (RUN_STOP fired mid-window by _do_eval_nb). + if should_stop or mlt.run_stopped: + # MLPerf target reached: RUN_STOP already emitted; stop training. + break + + # Close the run-spanning block for the data-fraction / no-eval case + # (idempotent; no-op under per-window blocks or after a convergence stop). + if not _per_window_blocks: + mlt.block_stop() + + # Final eval over the fixed user-holdout set (legacy final-window eval + # otherwise). Skipped if the MLPerf target already stopped the run mid-run. + if not mlt.run_stopped: + dataset.dataset.is_eval = True # pyre-ignore [16] + mlt.eval_start() + if eval_global_indices is not None: + final_metrics = _run_eval_window( + iter( + make_streaming_dataloader( + dataset=dataset, indices=eval_global_indices + ) + ), + label="eval_holdout@final", + ) + else: + final_metrics = _run_eval_window( + iter(make_streaming_dataloader(dataset=dataset, ts=num_train_ts)), + label="eval@final", + ) + mlt.eval_stop(final_metrics) + if rank == 0: + for k, v in final_metrics.items(): + print(f"{k}: {v}") + # End-of-run RUN_STOP: SUCCESS if final metric met target, else ABORTED. + mlt.finalize(final_metrics) diff --git a/recommendation_v4/generative_recommenders/dlrm_v3/utils.py b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py new file mode 100644 index 000000000..1e5d79993 --- /dev/null +++ b/recommendation_v4/generative_recommenders/dlrm_v3/utils.py @@ -0,0 +1,1806 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe +""" +mlperf dlrm_v3 inference benchmarking tool. +""" + +import contextlib +import logging +import os +import time +from pathlib import Path +from typing import Any, Callable, Dict, List, Optional + +import gin +import tensorboard # @manual=//tensorboard:lib # noqa: F401 - required implicit dep when using torch.utils.tensorboard +import torch +from generative_recommenders.dlrm_v3.datasets.dataset import DLRMv3RandomDataset +from generative_recommenders.dlrm_v3.datasets.kuairand import DLRMv3KuaiRandDataset +from generative_recommenders.dlrm_v3.datasets.movie_lens import DLRMv3MovieLensDataset +from generative_recommenders.dlrm_v3.datasets.synthetic_movie_lens import ( + DLRMv3SyntheticMovieLensDataset, +) +from generative_recommenders.dlrm_v3.datasets.synthetic_streaming import ( + DLRMv3SyntheticStreamingDataset, +) +from generative_recommenders.dlrm_v3.datasets.yambda import DLRMv3YambdaDataset +from generative_recommenders.modules.multitask_module import ( + MultitaskTaskType, + TaskConfig, +) +from torch.profiler import profile, profiler, ProfilerActivity # pyre-ignore [21] +from torch.utils.tensorboard import SummaryWriter +from torchrec.metrics.accuracy import AccuracyMetricComputation +from torchrec.metrics.auc import AUCMetricComputation, compute_auc +from torchrec.metrics.gauc import GAUCMetricComputation +from torchrec.metrics.mae import MAEMetricComputation +from torchrec.metrics.metrics_namespace import MetricName, MetricPrefix +from torchrec.metrics.mse import MSEMetricComputation +from torchrec.metrics.ne import NEMetricComputation +from torchrec.metrics.rec_metric import ( + MetricComputationReport, + RecMetricComputation, +) + + +class LifetimeAUCMetricComputation(AUCMetricComputation): + """AUC over a 10M-sample (~5.5 eval-window) trailing buffer; emits with the + LIFETIME prefix. + + NOTE: despite the name, this is NOT an uncapped since-step-0 AUC. The parent + ``AUCMetricComputation`` evicts the prediction/label/weight buffers down to + ``window_size`` in ``update()``; we instantiate it with + ``window_size=10_000_000``, so "lifetime" is a ~10M-sample trailing window. + Raise ``window_size`` (accepting unbounded buffer growth) if true cumulative + AUC is ever required. + + Checkpoint correctness: torchrec registers the PREDICTIONS/LABELS/WEIGHTS + buffers with ``persistent=False`` (so the default ``state_dict()`` drops + them) and tracks a separate ``self._num_samples`` counter. Without the + overrides below, every checkpoint resume would silently restart this metric + from an empty buffer. We therefore serialize the buffers AND ``_num_samples`` + explicitly; restoring ``_num_samples`` is mandatory, since leaving it at 0 + makes the next ``update()`` take the init-sentinel branch and desync the + windowed eviction. These buffers are per-rank-local (cross-rank gather only + happens transiently at compute time), so the checkpoint layer MUST persist + and restore them per-rank — see ``checkpoint.py``. + """ + + # Prefix used for the explicitly-serialized non-persistent buffers so the + # keys can't collide with any persistent state the parent might register. + _LIFETIME_KEY_PREFIX: str = "_lifetime_" + + def _compute(self) -> List[MetricComputationReport]: + from typing import cast as _cast + from torchrec.metrics.auc import LABELS, PREDICTIONS, WEIGHTS + return [ + MetricComputationReport( + name=MetricName.AUC, + metric_prefix=MetricPrefix.LIFETIME, + value=compute_auc( + self._n_tasks, + _cast(List[torch.Tensor], getattr(self, PREDICTIONS)), + _cast(List[torch.Tensor], getattr(self, LABELS)), + _cast(List[torch.Tensor], getattr(self, WEIGHTS)), + self._apply_bin, + ), + ) + ] + + def lifetime_sample_count(self) -> int: + """Current number of buffered samples (greppable for sanity logs).""" + return int(getattr(self, "_num_samples", 0)) + + def state_dict( + self, + destination: Optional[Dict[str, Any]] = None, + prefix: str = "", + keep_vars: bool = False, + ) -> Dict[str, Any]: + from torchrec.metrics.auc import LABELS, PREDICTIONS, WEIGHTS + + destination = super().state_dict( + destination=destination, prefix=prefix, keep_vars=keep_vars + ) + # The parent registers these buffers persistent=False, so they are absent + # from `destination`. Concatenate each buffer list to one (n_tasks, N) + # tensor and serialize it alongside the sample counter. + for attr in (PREDICTIONS, LABELS, WEIGHTS): + buf = getattr(self, attr) + if isinstance(buf, (list, tuple)) and len(buf) > 0: + flat = torch.cat([t for t in buf], dim=-1) + elif isinstance(buf, torch.Tensor): + flat = buf + else: + flat = torch.empty(0) + destination[prefix + self._LIFETIME_KEY_PREFIX + attr] = ( + flat.detach().cpu().clone() + ) + destination[prefix + self._LIFETIME_KEY_PREFIX + "num_samples"] = ( + torch.tensor(int(getattr(self, "_num_samples", 0)), dtype=torch.long) + ) + return destination + + def load_state_dict( + self, + state_dict: Dict[str, Any], + strict: bool = True, + ) -> Any: + from torchrec.metrics.auc import LABELS, PREDICTIONS, WEIGHTS + + # Copy so we can strip our custom keys before delegating to the parent + # (whose strict load would otherwise reject them as unexpected). + remaining = dict(state_dict) + saved_bufs: Dict[str, torch.Tensor] = {} + for attr in (PREDICTIONS, LABELS, WEIGHTS): + key = self._LIFETIME_KEY_PREFIX + attr + if key in remaining: + saved_bufs[attr] = remaining.pop(key) + num_key = self._LIFETIME_KEY_PREFIX + "num_samples" + saved_num = remaining.pop(num_key, None) + + result = super().load_state_dict(remaining, strict=strict) + + if saved_bufs: + # Device of the live (init-sentinel) buffers; keep restored buffers + # co-located so subsequent update()/compute() stay on-device. + existing = getattr(self, PREDICTIONS) + dev = ( + existing[0].device + if isinstance(existing, (list, tuple)) and len(existing) > 0 + else torch.device("cpu") + ) + for attr, val in saved_bufs.items(): + setattr(self, attr, [val.to(dev)]) + if saved_num is not None: + self._num_samples = int(saved_num.item()) + return result + + +# Sentinel "window size" used for the FRESH eval metrics so torchrec's windowed +# eviction never fires within a single eval pass (the per-pass reset bounds the +# buffer to exactly one full holdout pass). 1<<60 is far above any realistic +# per-rank sample count and avoids sys.maxsize overflow inside torchrec math. +UNBOUNDED_WINDOW: int = 1 << 60 + + +class BinnedCumulativeAUC(RecMetricComputation): + """Cumulative AUC via a fixed-resolution score histogram (LIFETIME prefix). + + Global AUC is a rank statistic, so it has no fixed-size additive sufficient + statistic the way NE/Accuracy do - exact cumulative AUC otherwise needs every + (score, label) pair retained and sorted (the buffer-based ``AUCMetricComputation`` + / ``LifetimeAUCMetricComputation``). Instead we keep two weighted histograms of + positive/negative mass per score bin. This gives an AUC exact up to bin width + with O(num_bins) memory that does NOT grow with sample count, and - because + histograms are additive - cross-rank sync is a cheap all-reduce (dist_reduce_fx + "sum") rather than all-gathering millions of predictions. The state is truly + cumulative across all eval passes (never evicted, never reset on eval). + + Predictions MUST be probabilities in [0, 1] (the same tensor feeds NE, which + requires probabilities; the model applies sigmoid in multitask_module). Values + are clamped into [0, 1] defensively. + """ + + def __init__(self, *args, num_bins: int = 100_000, **kwargs) -> None: + # window_size is irrelevant here (no windowed state); pass through. + super().__init__(*args, **kwargs) + self._num_bins: int = int(num_bins) + self._add_state( + "pos_hist", + torch.zeros((self._n_tasks, self._num_bins), dtype=torch.float64), + add_window_state=False, + dist_reduce_fx="sum", + persistent=True, + ) + self._add_state( + "neg_hist", + torch.zeros((self._n_tasks, self._num_bins), dtype=torch.float64), + add_window_state=False, + dist_reduce_fx="sum", + persistent=True, + ) + + def cumulative_sample_count(self) -> int: + """Total weighted samples in the histograms (greppable for sanity logs).""" + return int((self.pos_hist.sum() + self.neg_hist.sum()).item()) + + def update( + self, + *, + predictions: Optional[torch.Tensor], + labels: torch.Tensor, + weights: Optional[torch.Tensor], + **kwargs: Dict[str, Any], + ) -> None: + if predictions is None or weights is None: + raise ValueError( + "BinnedCumulativeAUC.update requires predictions and weights" + ) + preds = predictions.float().clamp_(0.0, 1.0) # (n_tasks, n_examples) + labels = labels.float() + weights = weights.float() + # Bin index per example; the top edge (p==1.0) folds into the last bin. + idx = (preds * self._num_bins).long().clamp_(0, self._num_bins - 1) + pos_w = (weights * labels).to(self.pos_hist.dtype) + neg_w = (weights * (1.0 - labels)).to(self.neg_hist.dtype) + self.pos_hist.scatter_add_(1, idx, pos_w) + self.neg_hist.scatter_add_(1, idx, neg_w) + + def _compute(self) -> List[MetricComputationReport]: + # By compute() time torchmetrics has all-reduced (summed) the histograms + # across ranks, so these are the global per-bin masses. + pos = self.pos_hist # (n_tasks, num_bins) + neg = self.neg_hist + total_pos = pos.sum(dim=1) + total_neg = neg.sum(dim=1) + # Lower bin index == lower score. A positive in bin b outranks every + # negative in bins < b (exclusive prefix sum), and ties in bin b score + # 0.5. AUC = sum_b pos_b * (neg_below_b + 0.5*neg_b) / (P * N). + neg_below = torch.cumsum(neg, dim=1) - neg + numerator = (pos * (neg_below + 0.5 * neg)).sum(dim=1) + denom = total_pos * total_neg + auc = torch.where( + denom > 0, + numerator / denom, + torch.full_like(numerator, 0.5), + ).to(torch.float32) + return [ + MetricComputationReport( + name=MetricName.AUC, + metric_prefix=MetricPrefix.LIFETIME, + value=auc, + ) + ] + + +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger("utils") + + +def _trim_warmup_from_trace(path: str, keep_n_active: int) -> None: + """Post-process a chrome trace to drop events from WARMUP-phase steps. + + torch.profiler captures events during BOTH the WARMUP and RECORD phases + of a schedule and writes them all to the exported trace. There is no + built-in flag to exclude WARMUP from the export. We approximate it by: + + 1) Finding all ``ProfilerStep#N`` spans in the file. + 2) Keeping only the last ``keep_n_active`` of them (sorted by start + timestamp) as the "active" range. + 3) Filtering ``traceEvents`` to events whose ``ts`` falls inside that + range. Metadata events (``ph='M'``) are always preserved. + + Mutates the file in place. + """ + import json as _json + with open(path) as f: + d = _json.load(f) + events = d.get("traceEvents", []) + + # ProfilerStep spans mark training-step boundaries; we filter by their + # time ranges rather than by name index because step numbering can offset + # between schedule_fn argument and the value printed in the trace. + # torch.profiler emits one ProfilerStep#N span per CPU thread that ran + # during that step, so dedupe by name first so "5 active steps" means + # 5 distinct step numbers, not 5 spans. + name_to_span: Dict[str, tuple] = {} + for e in events: + nm = e.get("name", "") + if "ProfilerStep" not in nm or e.get("ph") != "X" or "ts" not in e: + continue + ts = e["ts"] + end = ts + e.get("dur", 0) + prev = name_to_span.get(nm) + if prev is None: + name_to_span[nm] = (ts, end) + else: + name_to_span[nm] = (min(prev[0], ts), max(prev[1], end)) + if len(name_to_span) <= keep_n_active: + return + sorted_spans = sorted(name_to_span.values()) + active = sorted_spans[-keep_n_active:] + t_start = min(s for s, _ in active) + t_end = max(e for _, e in active) + + def _keep(e: dict) -> bool: + if e.get("ph") == "M": + return True + ts = e.get("ts") + if ts is None: + return True + return t_start <= ts < t_end + + kept = [e for e in events if _keep(e)] + d["traceEvents"] = kept + with open(path, "w") as f: + _json.dump(d, f) + logger.warning( + f"Trimmed WARMUP events from {path}: {len(events):,} -> {len(kept):,} " + f"(kept active range [{t_start:.0f}, {t_end:.0f}] us)" + ) + + +# GPU activity categories used to detect GPU stream rows and their busy time. +_GPU_KERNEL_CATS = frozenset({"kernel", "gpu_memcpy", "gpu_memset"}) + + +def _is_rocm() -> bool: + """True on ROCm/AMD builds (``torch.version.hip`` set), False on CUDA/B200. + + The ProfilerStep-layout normalization and the sub-us kernel de-overlap are + workarounds for how roctracer projects annotations/kernels onto HIP streams; + CUDA/CUPTI traces don't have those artifacts, so these passes must be skipped + on NVIDIA to avoid touching otherwise-correct traces. + """ + return getattr(torch.version, "hip", None) is not None + + +def _normalize_profilerstep_layout(path: str) -> None: + """Collapse fragmented GPU-side ``ProfilerStep#N`` spans into one span/step. + + ``torch.profiler`` emits ``ProfilerStep#N`` as a CPU ``user_annotation`` that + Kineto projects onto the GPU timeline as ``gpu_user_annotation`` spans. On + CUDA the blocking H2D copy shares the compute stream, so each step projects + onto a single GPU stream and renders as one full-width span. On ROCm a + blocking H2D copy lands on HIP's null stream (a different stream than the + non-null compute stream), so the step splits across two GPU rows and looks + truncated in Perfetto — a pure rendering artifact (every kernel is still + captured, and the underlying GPU is busy for the whole step). + + This rewrites each per-step GPU ``ProfilerStep`` annotation to a single span + on the rank's busiest (compute) GPU stream, covering the kernel extent inside + that step's CPU window. Works on a raw per-rank trace (GPU streams are tids + under one pid) by keying the busiest stream on ``(pid, tid)``. No-op when the + annotation already lives on a single GPU stream (the CUDA case), so it is + safe to run on every platform. Mutates the file in place. + """ + import json as _json + + with open(path) as f: + d = _json.load(f) + events = d.get("traceEvents", []) + + # Per (pid,tid) GPU busy time -> identify the busiest = compute stream. + stream_busy: Dict[tuple, int] = {} + for e in events: + if e.get("ph") == "X" and e.get("cat") in _GPU_KERNEL_CATS: + dur = e.get("dur", 0) + if dur > 0: + key = (e.get("pid"), e.get("tid")) + stream_busy[key] = stream_busy.get(key, 0) + dur + if not stream_busy: + return + busiest = max(stream_busy, key=lambda k: stream_busy[k]) + + # Existing GPU-side ProfilerStep spans and the streams they sit on. + gpu_ps_streams = set() + template = None + for e in events: + if e.get("cat") == "gpu_user_annotation" and str( + e.get("name", "") + ).startswith("ProfilerStep"): + gpu_ps_streams.add((e.get("pid"), e.get("tid"))) + if template is None: + template = e + # No fragmentation (single stream or none) -> leave the trace untouched. + if len(gpu_ps_streams) <= 1: + return + + # CPU ProfilerStep windows: step name -> [min ts, max end]. + cpu_win: Dict[str, list] = {} + for e in events: + if ( + e.get("cat") == "user_annotation" + and e.get("ph") == "X" + and str(e.get("name", "")).startswith("ProfilerStep") + ): + ts = e.get("ts", 0) + end = ts + e.get("dur", 0) + w = cpu_win.get(e["name"]) + if w is None: + cpu_win[e["name"]] = [ts, end] + else: + w[0] = min(w[0], ts) + w[1] = max(w[1], end) + + # GPU kernel extents (any stream) for clamping each step's span. + gpu_kernels = [ + (e.get("ts", 0), e.get("ts", 0) + e.get("dur", 0)) + for e in events + if e.get("ph") == "X" + and e.get("cat") in _GPU_KERNEL_CATS + and e.get("dur", 0) > 0 + ] + + new_spans = [] + for sname, (cs, ce) in cpu_win.items(): + ks = [(ts, end) for ts, end in gpu_kernels if end > cs and ts < ce] + if not ks: + continue + gmin = min(ts for ts, _ in ks) + gmax = max(end for _, end in ks) + span = dict(template) if template else {} + span.update( + { + "ph": "X", + "cat": "gpu_user_annotation", + "name": sname, + "pid": busiest[0], + "tid": busiest[1], + "ts": gmin, + "dur": gmax - gmin, + "args": {"normalized_profilerstep": True}, + } + ) + new_spans.append(span) + + if not new_spans: + return + + out = [ + e + for e in events + if not ( + e.get("cat") == "gpu_user_annotation" + and str(e.get("name", "")).startswith("ProfilerStep") + ) + ] + dropped = len(events) - len(out) + out.extend(new_spans) + d["traceEvents"] = out + with open(path, "w") as f: + _json.dump(d, f) + logger.warning( + f"Normalized GPU ProfilerStep layout in {path}: dropped {dropped} " + f"fragmented span(s) across {len(gpu_ps_streams)} stream(s), wrote " + f"{len(new_spans)} span(s) on busiest stream pid={busiest[0]} " + f"tid={busiest[1]}" + ) + + +def _deoverlap_gpu_slices(path: str, max_snap_us: float = 5.0) -> None: + """Remove sub-microsecond kernel overlaps that break Perfetto's renderer. + + Perfetto draws all ``ph=="X"`` slices on a single track (one ``(pid, tid)``) + as a strict nested stack ordered by start time: a slice that *opens* while a + previous slice on the same track is still open is treated as that slice's + child and is **clipped to the parent's end**. ROCm's roctracer reports + per-stream kernel timestamps at ns granularity, so two back-to-back kernels + on the same compute stream occasionally overlap by a fraction of a + microsecond (e.g. an 88 ns ``elementwise`` epilogue ending 0.075 us *after* + the next 21 ms ``_hstu_attn_bwd`` kernel begins). Perfetto then nests the + long kernel inside the tiny one and clips it to a sub-pixel sliver, so the + kernel "disappears" from the timeline even though it is fully present in the + JSON. + + This pulls each slice's end back to just *before* the next slice's start + whenever they overlap by less than ``max_snap_us`` (a measurement artifact, + not real concurrency — kernels on one stream are serialized), leaving genuine + nesting (a small kernel fully contained in a larger one) untouched. The + adjustment is sub-microsecond and does not change any reported duration + meaningfully. Mutates the file in place; best-effort. + + Critically, the slices are separated by a tiny ``_GAP_US`` (~1 ns) rather + than snapped to an *exactly equal* end==start timestamp. A coincident + end==start is just as fatal as an overlap in Perfetto: it nests the next + slice inside the previous one and clips it to zero width (this is the ~1 ns + gap that roctracer leaves between cleanly-rendered back-to-back kernels). So + we also fix exact-touch (``a_end == b.ts``) boundaries, not just overlaps. + """ + import json as _json + from collections import defaultdict + + # ~1 ns. Matches the natural inter-kernel gap roctracer leaves between + # back-to-back kernels that Perfetto already renders correctly. Must be + # strictly > 0 so end != start after the nudge. + _GAP_US = 0.001 + + with open(path) as f: + d = _json.load(f) + events = d.get("traceEvents", []) + + tracks: Dict[tuple, list] = defaultdict(list) + for e in events: + if ( + e.get("ph") == "X" + and e.get("cat") in _GPU_KERNEL_CATS + and e.get("dur", 0) > 0 + ): + tracks[(e.get("pid"), e.get("tid"))].append(e) + + snapped = 0 + max_clip = 0.0 + for sl in tracks.values(): + # Sort by start, then longest-first so a container precedes the slices + # it nests; consecutive pairs are then either disjoint, properly nested, + # or a tiny artifact overlap. + sl.sort(key=lambda e: (e["ts"], -e["dur"])) + for i in range(len(sl) - 1): + a = sl[i] + b = sl[i + 1] + a_end = a["ts"] + a["dur"] + b_end = b["ts"] + b["dur"] + # Touching (a_end == b.ts) or partial overlap (a ends inside b) both + # break rendering; true containment (a_end >= b_end) is valid nesting + # and is left alone. + if b["ts"] <= a_end < b_end: + desired_end = b["ts"] - _GAP_US + clip = a_end - desired_end + if a["ts"] < desired_end and 0 < clip < max_snap_us: + a["dur"] = desired_end - a["ts"] + snapped += 1 + if clip > max_clip: + max_clip = clip + + if snapped: + with open(path, "w") as f: + _json.dump(d, f) + logger.warning( + f"De-overlapped GPU slices in {path}: snapped {snapped} sub-us " + f"overlap(s) (max {max_clip:.3f}us) so Perfetto renders every kernel" + ) + + +def _deoverlap_gpu_annotations(path: str, max_snap_us: float = 5.0) -> None: + """Separate touching/overlapping *sibling* GPU annotations so Perfetto draws + each one full width (the B200-style stacked layout). + + Same root cause as :func:`_deoverlap_gpu_slices`, but at the annotation + boundary instead of the kernel boundary. The forward/backward phase + annotations Kineto projects onto the GPU stream (``## item_forward ##``, + ``## user_forward ##``, ``## multitask_module ##``, the ``## stu_* ##`` + pairs, ...) are emitted as a chain of siblings laid end-to-end: each is meant + to end exactly where the next begins. Perfetto stores timestamps as int64 ns, + and the absolute step timestamps are ~5.4e12 us where a float64's quantum is + already ~1 ns, so a sibling boundary that should be coincident instead lands + a few ns off. When the earlier sibling's end falls *at or after* the next + sibling's start, Perfetto nests the next sibling inside it and clips it to a + sub-pixel sliver — so e.g. the 100+ ms ``## user_forward ##`` span vanishes on + some ranks/steps and renders on others purely by rounding luck. + + Unlike kernels (all flat on one stream), annotations form a real nesting + hierarchy — ``## user_forward ##`` legitimately *contains* the ``## stu_* ##`` + spans and their kernels — so this cannot blindly snap consecutive slices. It + walks the per-track slice stack (sorted by start, longest-first) and only + snaps a slice ``a`` back when the next slice ``b`` is **not** contained in it + (``b`` extends beyond ``a``'s end), i.e. they are siblings rather than + parent/child. Real containment is left untouched, and a snap is skipped if it + would clip into ``a``'s own descendants (kernels or child annotations). + Mutates the file in place; best-effort. Run after :func:`_deoverlap_gpu_slices` + so kernel boundaries are already clean. + """ + import json as _json + from collections import defaultdict + + # ~2 ns. The annotation boundaries sit at ~5.4e12 us where a float64's + # quantum is ~0.98 ns, so a 1 ns nudge can round back onto the neighbour's + # timestamp (an exact touch, which Perfetto still nests+clips). 2 ns (~2 + # quanta) reliably separates them and is still far below any visible width. + _GAP_US = 0.002 + + with open(path) as f: + d = _json.load(f) + events = d.get("traceEvents", []) + + # Stack the full per-track hierarchy over BOTH kernels and annotations so a + # parent annotation knows the extent of its descendants (the snap guard), + # but only annotation slices are ever trimmed. + _ANN = "gpu_user_annotation" + tracks: Dict[tuple, list] = defaultdict(list) + for e in events: + if ( + e.get("ph") == "X" + and e.get("dur", 0) > 0 + and (e.get("cat") in _GPU_KERNEL_CATS or e.get("cat") == _ANN) + ): + tracks[(e.get("pid"), e.get("tid"))].append(e) + + snapped = 0 + max_clip = 0.0 + for sl in tracks.values(): + # Longest-first on ties so a container precedes the slices it nests. + sl.sort(key=lambda e: (e["ts"], -e["dur"])) + # Each frame: [event, max_descendant_end]. The stack holds the chain of + # currently-open ancestors for the slice being placed. + stack: list = [] + for b in sl: + b_ts = b["ts"] + b_end = b_ts + b["dur"] + while stack: + a = stack[-1][0] + a_end = a["ts"] + a["dur"] + if a_end < b_ts: + # a closed strictly before b begins -> disjoint sibling, pop. + frame = stack.pop() + eff = frame[0]["ts"] + frame[0]["dur"] + if stack: + stack[-1][1] = max(stack[-1][1], eff, frame[1]) + continue + if a_end < b_end: + # b starts at/inside a but extends past a's end => they are + # siblings (not parent/child), and a's tail nests+clips b in + # Perfetto. Snap a's end to just before b. This fires for both + # annotation tails (## item_forward ## overhanging + # ## user_forward ##) and kernel tails that straddle an + # annotation boundary (a layer-norm kernel ending a few ns + # past the start of the next phase span) -- both are sub-us + # roctracer/rounding artifacts, since kernels on one stream + # are serialized and phase spans are sequential. + desired_end = b_ts - _GAP_US + clip = a_end - desired_end + # Guard: only snap when a's deepest descendant ends at or + # before b's start. If a child (kernel or nested span) + # actually extends *past* b.ts, trimming a wouldn't fix b's + # clipping (the child would still nest b) and could drop a + # real child into b's territory, so leave it. A descendant + # ending exactly at the boundary is itself rounding noise and + # is clipped by <=1 ns, which is fine. + if ( + a["ts"] < desired_end + and stack[-1][1] <= b_ts + and 0 < clip < max_snap_us + ): + a["dur"] = desired_end - a["ts"] + snapped += 1 + if clip > max_clip: + max_clip = clip + frame = stack.pop() + eff = frame[0]["ts"] + frame[0]["dur"] + if stack: + stack[-1][1] = max(stack[-1][1], eff, frame[1]) + continue + # a_end >= b_end: a fully contains b -> b is a child, stop. + break + stack.append([b, b_ts]) + + if snapped: + with open(path, "w") as f: + _json.dump(d, f) + logger.warning( + f"De-overlapped GPU annotations in {path}: snapped {snapped} sub-us " + f"sibling overlap(s) (max {max_clip:.3f}us) so Perfetto renders every " + f"annotation full width" + ) + + +def _on_trace_ready_fn( + rank: Optional[int] = None, + trace_dir: str = "/tmp/dlrm_v3_traces", + keep_n_active: Optional[int] = None, + trace_steps: Optional[List[int]] = None, +) -> Callable[[torch.profiler.profile], None]: + """Create the on_trace_ready callback that exports a chrome trace to disk. + + Filename follows ``trace_step{step}_rank{rank}.json`` so multi-rank + captures don't collide and ``scripts/stitch_traces.py`` can merge them + by step number. + + The ``{step}`` label: + + * If ``trace_steps`` is provided (multi-window mode), the Nth callback + invocation labels its file with ``trace_steps[N]`` -- i.e. the + user-requested step that triggered the window. This is the most + intuitive labelling. + * Otherwise falls back to ``p.step_num`` (torch.profiler's internal + counter at trigger time, off by ~warmup+active from the schedule + arg). + + If ``keep_n_active`` is set, the exported file is post-processed to keep + only the last N ProfilerStep-spans worth of events (i.e. drop WARMUP). + """ + state = {"fire_count": 0} + + def handle_fn(p: torch.profiler.profile) -> None: + os.makedirs(trace_dir, exist_ok=True) + if trace_steps: + i = state["fire_count"] + step_label = ( + trace_steps[i] if i < len(trace_steps) else getattr(p, "step_num", 0) + ) + else: + step_label = getattr(p, "step_num", 0) + state["fire_count"] += 1 + rank_str = f"_rank{rank}" if rank is not None else "" + file_name = f"trace_step{step_label}{rank_str}.json" + path = os.path.join(trace_dir, file_name) + logger.warning( + p.key_averages(group_by_input_shape=True).table( + sort_by="self_cuda_time_total" + ) + ) + # Tracing is best-effort: a write/trim failure (permissions, disk full, + # malformed export) must never crash the training run. Degrade to a + # warning so the loop continues — especially important since streaming + # enables output_trace by default. + try: + p.export_chrome_trace(path) + logger.warning(f"Trace written to: {path}") + if keep_n_active is not None and keep_n_active > 0: + _trim_warmup_from_trace(path, keep_n_active) + # ROCm/AMD-only rendering fixes. CUDA/CUPTI (e.g. B200) traces don't + # exhibit the fragmented-ProfilerStep or sub-us kernel-overlap + # artifacts, so skip entirely on NVIDIA to avoid touching otherwise + # correct traces. Best-effort like trim above. + if _is_rocm(): + # Normalize the GPU-side ProfilerStep layout so ROCm traces + # render with one full-width step span per stream like CUDA. + _normalize_profilerstep_layout(path) + # Snap roctracer's sub-us kernel overlaps so Perfetto doesn't + # mis-nest and hide long kernels. + _deoverlap_gpu_slices(path) + # Same fix at the annotation-sibling boundary so phase spans + # (## user_forward ##, ## stu_* ##, ...) render full width. + _deoverlap_gpu_annotations(path) + except Exception as exc: + logger.warning(f"Trace export/trim failed for {path}: {exc!r} (skipping)") + + return handle_fn + + +def profiler_or_nullcontext( + enabled: bool, with_stack: bool, trace_dir: str = "/tmp/dlrm_v3_traces" +): + """One-shot profile context for ad-hoc captures (no scheduling).""" + return ( + profile( + # pyre-fixme[16]: Module `profiler` has no attribute `ProfilerActivity`. + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], + on_trace_ready=_on_trace_ready_fn(trace_dir=trace_dir), + with_stack=with_stack, + ) + if enabled + else contextlib.nullcontext() + ) + + +def _multi_window_schedule( + trace_steps, + warmup: int, + active: int, +): + """Custom schedule that profiles around each step in ``trace_steps``. + + Step s gets: + [s - warmup, s) -> WARMUP + [s, s + active - 1) -> RECORD + s + active - 1 -> RECORD_AND_SAVE + """ + windows = [(s - warmup, s, s + active) for s in sorted(trace_steps)] + + def schedule_fn(step: int) -> torch.profiler.ProfilerAction: + for warmup_start, active_start, active_end in windows: + if warmup_start <= step < active_start: + return torch.profiler.ProfilerAction.WARMUP + if active_start <= step < active_end - 1: + return torch.profiler.ProfilerAction.RECORD + if step == active_end - 1: + return torch.profiler.ProfilerAction.RECORD_AND_SAVE + return torch.profiler.ProfilerAction.NONE + + return schedule_fn + + +@gin.configurable +class Profiler: + """Scheduled torch.profiler wrapper that writes Chrome traces to disk. + + Two modes (set via gin): + + * Single window (default): ``wait=10, warmup=20, active=50, repeat=1``. + Captures one contiguous window starting after ``wait`` steps. + * Multi-window: ``trace_steps=[500, 1000, 5000]`` (overrides wait+repeat). + Captures a separate window around each listed step. + + All knobs are gin-tunable, e.g. in a gin file:: + + Profiler.trace_dir = "/path/to/results/exp42/trace" + Profiler.trace_steps = [500, 1000, 5000] + Profiler.warmup = 5 + Profiler.active = 10 + """ + + def __init__( + self, + rank: int, + active: int = 50, + wait: int = 10, + warmup: int = 20, + repeat: int = 1, + trace_steps: Optional[List[int]] = None, + trace_dir: str = "/tmp/dlrm_v3_traces", + trim_warmup: bool = True, + record_shapes: bool = True, + profile_memory: bool = False, + with_stack: bool = False, + with_flops: bool = False, + with_modules: bool = False, + ) -> None: + self.rank = rank + self.trace_dir = trace_dir + if trace_steps: + sched = _multi_window_schedule(trace_steps, warmup, active) + else: + sched = torch.profiler.schedule( + wait=wait, warmup=warmup, active=active, repeat=repeat + ) + keep_n = active if trim_warmup else None + self._profiler: profiler.profile = torch.profiler.profile( + schedule=sched, + on_trace_ready=_on_trace_ready_fn( + self.rank, trace_dir, keep_n, trace_steps + ), + # pyre-fixme[16]: Module `profiler` has no attribute `ProfilerActivity`. + activities=[ProfilerActivity.CPU, ProfilerActivity.CUDA], + record_shapes=record_shapes, + profile_memory=profile_memory, + with_stack=with_stack, + with_flops=with_flops, + with_modules=with_modules, + ) + + def step(self) -> None: + """Advance the profiler to the next step.""" + self._profiler.step() + + +class _NoOpSummaryWriter: + """Drop-in stand-in for SummaryWriter used when TensorBoard is disabled + (empty ``tensorboard_log_path``). All scalar writes become no-ops so the + metrics path (compute + text-log + ``.metrics.jsonl`` sinks) runs unchanged + and never crashes on TB file I/O. The shared ``/apps`` tfevents writer was a + crash source under transient filer ``Errno 121`` (Remote I/O) errors, and + nothing we consume reads TensorBoard — eval-window AUCs come from the JSONL + sink and the text log.""" + + def add_scalar(self, *args, **kwargs) -> None: + pass + + def flush(self) -> None: + pass + + def close(self) -> None: + pass + + +@gin.configurable +class MetricsLogger: + """ + Logger for tracking and computing recommendation metrics. + + Supports both classification metrics (NE, Accuracy, GAUC) and regression + metrics (MSE, MAE) based on multitask configuration. + + Args: + multitask_configs: List of task configurations defining metric types. + batch_size: Batch size for metric computation. + window_size: Window size for running metric aggregation. + device: Device to place metric tensors on. + rank: Process rank for distributed training. + tensorboard_log_path: Optional path for TensorBoard logging. + """ + + def __init__( + self, + multitask_configs: List[TaskConfig], + batch_size: int, + window_size: int, + device: torch.device, + rank: int, + tensorboard_log_path: str = "", + world_size: int = 1, + auc_threshold: Optional[float] = None, + num_flops_per_sample: float = 0.0, + gpu_peak_flops: float = 0.0, + model: Optional[torch.nn.Module] = None, + eval_cumulative: bool = False, + cumulative_auc_bins: int = 100_000, + train_lifetime_auc_mode: str = "binned", + eval_lifetime_auc_mode: str = "binned", + lifetime_auc_window: int = 10_000_000, + ) -> None: + # tflops/mfu reporting state (optional — when both num_flops_per_sample + # and gpu_peak_flops are set, the train perf line gains tflops_algo/gpu, + # mfu, tflops_real/gpu, hfu, fill. The jagged ("real") numbers come + # from `model._last_jagged_flops_per_sample` stashed by DlrmHSTU.main_forward. + self._num_flops_per_sample: float = max(0.0, float(num_flops_per_sample)) + self._gpu_peak_flops: float = max(0.0, float(gpu_peak_flops)) + self._model_ref: Optional[torch.nn.Module] = model + if rank == 0 and self._num_flops_per_sample > 0 and self._gpu_peak_flops > 0: + logger.info( + f"FLOPS reporting enabled: {self._num_flops_per_sample / 1e9:.1f} " + f"GFLOP/sample (dense fwd+bwd), GPU peak {self._gpu_peak_flops / 1e12:.0f} TFLOPS" + ) + self.multitask_configs: List[TaskConfig] = multitask_configs + all_classification_tasks: List[str] = [ + task.task_name + for task in self.multitask_configs + if task.task_type != MultitaskTaskType.REGRESSION + ] + all_regression_tasks: List[str] = [ + task.task_name + for task in self.multitask_configs + if task.task_type == MultitaskTaskType.REGRESSION + ] + assert all_classification_tasks + all_regression_tasks == [ + task.task_name for task in multitask_configs + ] + self.task_names: List[str] = all_classification_tasks + all_regression_tasks + + # Eval metric semantics: + # eval_cumulative=False (default, legacy / static / non-streaming eval): + # a single eval set with the configured window_size, including a + # lifetime AUC. Unchanged behavior. + # eval_cumulative=True (streaming fixed-holdout eval): a FRESH eval set + # (window_size=UNBOUNDED, reset each pass -> per-pass full-holdout + # "window_*") PLUS a CUMULATIVE set ("eval_cum", never reset -> + # "lifetime_*"). NE/Accuracy/GAUC are cumulative for free via their + # persistent scalar sums; AUC cumulative uses the selected backend. + # + # Lifetime-AUC backend is configurable independently for train and eval: + # "binned" (default): BinnedCumulativeAUC - exact-cumulative AUC via an + # O(num_bins) score histogram (additive all-reduce, no unbounded + # buffer, memory independent of #samples/#windows). + # "capped": LifetimeAUCMetricComputation - AUC over a trailing buffer of + # `lifetime_auc_window` samples/rank (the legacy approach; per-rank + # buffer all-gathered at compute). + self._eval_cumulative: bool = eval_cumulative + self._cumulative_auc_bins: int = int(cumulative_auc_bins) + self._train_lifetime_auc_mode: str = str(train_lifetime_auc_mode) + self._eval_lifetime_auc_mode: str = str(eval_lifetime_auc_mode) + self._lifetime_auc_window: int = int(lifetime_auc_window) + n_cls = len(all_classification_tasks) + n_reg = len(all_regression_tasks) + + def _make_lifetime_auc(mode: str) -> RecMetricComputation: + if mode == "binned": + # window_size=0: no torchrec windowed state; histograms only. + return BinnedCumulativeAUC( + my_rank=rank, batch_size=batch_size, n_tasks=n_cls, + window_size=0, num_bins=self._cumulative_auc_bins, + ).to(device) + if mode == "capped": + return LifetimeAUCMetricComputation( + my_rank=rank, batch_size=batch_size, n_tasks=n_cls, + window_size=self._lifetime_auc_window, + ).to(device) + raise ValueError( + f"lifetime_auc_mode must be 'binned' or 'capped', got {mode!r}" + ) + + def _make_class(ws: int, lifetime_mode: Optional[str]) -> List[RecMetricComputation]: + mets: List[RecMetricComputation] = [ + NEMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=ws).to(device), + AccuracyMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=ws).to(device), + GAUCMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=ws).to(device), + AUCMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=ws).to(device), + ] + if lifetime_mode is not None: + mets.append(_make_lifetime_auc(lifetime_mode)) + return mets + + def _make_class_cumulative() -> List[RecMetricComputation]: + # NE/Accuracy/GAUC: cumulative via persistent lifetime sums (window + # value ignored at compute). AUC: selected lifetime backend. + return [ + NEMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=window_size).to(device), + AccuracyMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=window_size).to(device), + GAUCMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_cls, window_size=window_size).to(device), + _make_lifetime_auc(self._eval_lifetime_auc_mode), + ] + + def _make_reg(ws: int) -> List[RecMetricComputation]: + return [ + MSEMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_reg, window_size=ws).to(device), + MAEMetricComputation(my_rank=rank, batch_size=batch_size, n_tasks=n_reg, window_size=ws).to(device), + ] + + self.class_metrics: Dict[str, List[RecMetricComputation]] = {"train": [], "eval": []} + self.regression_metrics: Dict[str, List[RecMetricComputation]] = {"train": [], "eval": []} + if eval_cumulative: + self.class_metrics["eval_cum"] = [] + self.regression_metrics["eval_cum"] = [] + + if all_classification_tasks: + self.class_metrics["train"] = _make_class(window_size, lifetime_mode=self._train_lifetime_auc_mode) + if eval_cumulative: + self.class_metrics["eval"] = _make_class(UNBOUNDED_WINDOW, lifetime_mode=None) + self.class_metrics["eval_cum"] = _make_class_cumulative() + else: + self.class_metrics["eval"] = _make_class(window_size, lifetime_mode=self._eval_lifetime_auc_mode) + + if all_regression_tasks: + self.regression_metrics["train"] = _make_reg(window_size) + if eval_cumulative: + self.regression_metrics["eval"] = _make_reg(UNBOUNDED_WINDOW) + self.regression_metrics["eval_cum"] = _make_reg(window_size) + else: + self.regression_metrics["eval"] = _make_reg(window_size) + + self.global_step: Dict[str, int] = {"train": 0, "eval": 0} + # MLPerf `samples_count` progress unit: global trained samples, persisted + # alongside global_step so a resumed run continues the count. + self.cumulative_train_samples: int = 0 + # Whether the MLPerf run markers (RUN_START etc.) were already emitted for + # this logical run. Checkpointed so a resume relaunch knows the run is + # already open and does NOT re-emit INIT_START/RUN_START (the compliance + # checker requires EXACTLY_ONE); the resumed process continues the same + # event stream and emits the single RUN_STOP at convergence/end. + self.mlperf_run_started: bool = False + self._rank: int = int(rank) + # Optional MLPerf logger + LR accessor wired by the streaming loop (duck- + # typed to avoid a train-module import cycle); drives the train_loss event. + self.mlperf_logger: Optional[Any] = None + self.lr_getter: Optional[Callable[[], float]] = None + self.tb_logger: Optional[SummaryWriter] = None + if tensorboard_log_path != "": + self.tb_logger = SummaryWriter(log_dir=tensorboard_log_path, purge_step=0) + self.tb_logger.flush() + else: + # TB disabled: use a no-op writer so the existing call sites (and the + # `assert self.tb_logger is not None` in compute_and_log) keep working + # while no tfevents are written to the fragile shared filer. + self.tb_logger = _NoOpSummaryWriter() + + # Throughput / time-to-target tracking. Counters are train-only; eval + # samples are not relevant for headline samples/sec numbers. + self._world_size: int = max(1, int(world_size)) + self._auc_threshold: Optional[float] = auc_threshold + self._time_to_target_logged: bool = False + self._perf_t_start: float = time.perf_counter() + self._perf_t_window: float = self._perf_t_start + self._perf_steps_in_window: int = 0 + self._perf_total_samples: int = 0 + self._perf_samples_counter: torch.Tensor = torch.zeros( + 1, dtype=torch.long, device=device + ) + # Non-train wall-time to exclude from the train step-time window so + # `step_ms` reports the canonical per-step compute latency even when an + # interval coincides with eval or checkpointing. Categorized so the + # excluded time is also reportable (eval_ms / ckpt_ms) rather than just + # discarded. The trainer brackets eval/ckpt regions via + # pause_perf(cat)/resume_perf(cat); accumulators reset each train-perf log. + self._perf_excluded: Dict[str, float] = {"eval": 0.0, "ckpt": 0.0} + self._perf_pause: Dict[str, Optional[float]] = {} + + def pause_perf(self, category: str) -> None: + """Start excluding wall-time under `category` (e.g. "eval"/"ckpt") from + the train step-time window. Idempotent: a second pause without an + intervening resume is a no-op (keeps the earliest start).""" + if self._perf_pause.get(category) is None: + self._perf_pause[category] = time.perf_counter() + + def resume_perf(self, category: str) -> None: + """Stop excluding `category` and fold the elapsed interval into the + per-category accumulator. No-op if not currently paused.""" + t0 = self._perf_pause.get(category) + if t0 is not None: + self._perf_excluded[category] = ( + self._perf_excluded.get(category, 0.0) + + (time.perf_counter() - t0) + ) + self._perf_pause[category] = None + + @property + def auc_threshold(self) -> Optional[float]: + """Configured time-to-target AUC threshold (None if unset).""" + return self._auc_threshold + + @property + def all_metrics(self) -> Dict[str, List[RecMetricComputation]]: + """ + Get all metrics for train and eval modes. + + Returns: + Dictionary mapping mode ('train'/'eval') to list of metric computations. + """ + out = { + "train": self.class_metrics["train"] + self.regression_metrics["train"], + "eval": self.class_metrics["eval"] + self.regression_metrics["eval"], + } + if "eval_cum" in self.class_metrics or "eval_cum" in self.regression_metrics: + out["eval_cum"] = self.class_metrics.get( + "eval_cum", [] + ) + self.regression_metrics.get("eval_cum", []) + return out + + def update( + self, + predictions: torch.Tensor, + weights: torch.Tensor, + labels: torch.Tensor, + num_candidates: torch.Tensor, + mode: str = "train", + ) -> None: + """ + Update metrics with new batch of predictions and labels. + + Args: + predictions: Model prediction tensor. + weights: Sample weight tensor. + labels: Ground truth label tensor. + num_candidates: Number of candidates per sample (for GAUC). + mode: Either 'train' or 'eval'. + """ + # On eval, update BOTH the fresh set and the never-reset cumulative set + # (if enabled) from the same batch. + update_targets = list(self.all_metrics[mode]) + if mode == "eval" and "eval_cum" in self.all_metrics: + update_targets = update_targets + self.all_metrics["eval_cum"] + for metric in update_targets: + if isinstance(metric, GAUCMetricComputation): + metric.update( + predictions=predictions, + labels=labels, + weights=weights, + num_candidates=num_candidates, + ) + else: + metric.update( + predictions=predictions, + labels=labels, + weights=weights, + ) + self.global_step[mode] += 1 + if mode == "train": + # Accumulate on-device to avoid a per-step GPU->CPU sync; we read + # the counter only at compute_and_log boundaries. + self._perf_samples_counter += num_candidates.sum().to( + self._perf_samples_counter.dtype + ) + self._perf_steps_in_window += 1 + # MLPerf progress counter: per-rank sample count scaled by world size + # approximates global trained samples without an extra collective. + self.cumulative_train_samples += int(num_candidates.numel()) * self._world_size + + def compute(self, mode: str = "train") -> Dict[str, float]: + """ + Compute and return all metrics for the current window. + + Args: + mode: Either 'train' or 'eval'. + + Returns: + Dictionary mapping metric names to their computed values. + """ + all_computed_metrics = {} + + if mode == "eval" and "eval_cum" in self.all_metrics: + # Dual-set eval: `window_*` (fresh per-pass) from the reset-each-pass + # set; `lifetime_*` (cumulative across passes) from the never-reset + # set. Filter each set to the matching prefix, and drop GAUC's + # auxiliary `*_num_samples` reports. Key names are unchanged + # (`window_auc`, `lifetime_ne`, ...) so dashboards keep working. + def _emit( + metrics: List[RecMetricComputation], keep_prefix: str + ) -> None: + for metric in metrics: + for computed in metric.compute(): + pfx = str(computed.metric_prefix) + name = str(computed.name) + if pfx != keep_prefix or name.endswith("num_samples"): + continue + all_values = computed.value.cpu() + for i, task_name in enumerate(self.task_names): + if i >= len(all_values): + break + all_computed_metrics[f"metric/{pfx}{name}/{task_name}"] = ( + all_values[i] + ) + + _emit(self.all_metrics["eval"], "window_") + _emit(self.all_metrics["eval_cum"], "lifetime_") + else: + for metric in self.all_metrics[mode]: + computed_metrics = metric.compute() + for computed in computed_metrics: + all_values = computed.value.cpu() + for i, task_name in enumerate(self.task_names): + if i >= len(all_values): + break + key = f"metric/{str(computed.metric_prefix) + str(computed.name)}/{task_name}" + all_computed_metrics[key] = all_values[i] + + logger.info( + f"{mode} - Step {self.global_step[mode]} metrics: {all_computed_metrics}" + ) + return all_computed_metrics + + def _global_mean_loss(self, loss_terms: Dict[str, torch.Tensor]) -> float: + """Cross-rank mean of the summed per-task losses. + + The 1-element all-reduce MUST run on every rank in lockstep; callers gate + it on a rank-identical counter (global_step / a deterministic frequency) + so it cannot desync. + """ + loss_t = torch.stack( + [v.detach().float().sum() for v in loss_terms.values()] + ).sum() + if torch.distributed.is_available() and torch.distributed.is_initialized(): + torch.distributed.all_reduce(loss_t, op=torch.distributed.ReduceOp.SUM) + loss_t = loss_t / self._world_size + return float(loss_t) + + def maybe_log_mlperf_train_loss( + self, aux_losses: Dict[str, torch.Tensor], every: int + ) -> None: + """Emit the MLPerf ``train_loss`` event on its OWN cadence. + + Decoupled from the console/TB metric cadence (``compute_and_log``): call + this every train step with the just-computed ``aux_losses`` and the + desired interval ``every`` (in global train steps). The cross-rank loss + all-reduce only fires on the cadence, gated by ``global_step["train"]`` + which is incremented identically on all ranks in ``update()`` — so the + collective stays in lockstep. No-op when MLPerf logging is disabled + (``mlperf_logger is None`` on every rank) or ``every <= 0``. + """ + if self.mlperf_logger is None or every <= 0 or not aux_losses: + return + if self.global_step["train"] % every != 0: + return + train_loss = self._global_mean_loss(aux_losses) + c = self.mlperf_logger.constants + md: Dict[str, Any] = {c.SAMPLES_COUNT: self.cumulative_train_samples} + if self.lr_getter is not None: + try: + md["lr"] = float(self.lr_getter()) + except Exception: + pass + self.mlperf_logger.event( + key=getattr(c, "TRAIN_LOSS", "train_loss"), + value=train_loss, + metadata=md, + ) + + def compute_and_log( + self, + mode: str = "train", + additional_logs: Optional[Dict[str, Dict[str, torch.Tensor]]] = None, + ) -> Dict[str, float]: + """ + Compute metrics and log to TensorBoard. + + Args: + mode: Either 'train' or 'eval'. + additional_logs: Optional additional data to log. + + Returns: + Dictionary mapping metric names to their computed values. + + Raises: + AssertionError: If TensorBoard logger is not configured. + """ + assert self.tb_logger is not None + all_computed_metrics = self.compute(mode) + for k, v in all_computed_metrics.items(): + self.tb_logger.add_scalar( # pyre-ignore [16] + f"{mode}_{k}", + v, + global_step=self.global_step[mode], + ) + + if additional_logs is not None: + for tag, data in additional_logs.items(): + for data_name, data_value in data.items(): + self.tb_logger.add_scalar( + f"{tag}/{mode}_{data_name}", + data_value.detach().clone().cpu(), + global_step=self.global_step[mode], + ) + + # Global (cross-rank mean) train loss to console/TB at the metric-logging + # cadence. The 1-element all-reduce runs on every rank in lockstep, so it + # cannot desync. The MLPerf `train_loss` EVENT is emitted separately via + # ``maybe_log_mlperf_train_loss`` so its cadence can be tuned independently + # of this console/TB cadence (see that method). + if mode == "train" and additional_logs is not None and "losses" in additional_logs: + loss_terms = additional_logs["losses"] + if loss_terms: + train_loss = self._global_mean_loss(loss_terms) + self.tb_logger.add_scalar( + "train_loss", train_loss, global_step=self.global_step["train"] + ) + if self._rank == 0: + logger.info( + f"train - Step {self.global_step['train']} " + f"train_loss={train_loss:.5f}" + ) + + # Throughput metrics (train only). One GPU->CPU sync per call. + if mode == "train" and self._perf_steps_in_window > 0: + now = time.perf_counter() + wall_dt = max(now - self._perf_t_window, 1e-6) + # Subtract bracketed eval/checkpoint wall-time so step_ms / sps / + # MFU reflect canonical train-step compute, not eval+ckpt stalls + # that happened to land in this window. The excluded time is also + # surfaced separately below (eval_ms / ckpt_ms) rather than discarded. + eval_s = self._perf_excluded.get("eval", 0.0) + ckpt_s = self._perf_excluded.get("ckpt", 0.0) + dt = max(wall_dt - eval_s - ckpt_s, 1e-6) + n_samples = int(self._perf_samples_counter.item()) + self._perf_total_samples += n_samples + local_sps = n_samples / dt + global_sps = local_sps * self._world_size + step_ms = dt * 1000.0 / self._perf_steps_in_window + wall_step_ms = wall_dt * 1000.0 / self._perf_steps_in_window + eval_ms = eval_s * 1000.0 + ckpt_ms = ckpt_s * 1000.0 + elapsed = now - self._perf_t_start + step = self.global_step["train"] + self.tb_logger.add_scalar( + "perf/train_samples_per_sec_local", local_sps, global_step=step + ) + self.tb_logger.add_scalar( + "perf/train_samples_per_sec_global", global_sps, global_step=step + ) + self.tb_logger.add_scalar( + "perf/train_step_time_ms", step_ms, global_step=step + ) + # Inclusive (old-semantics) per-step wall time and the eval/ckpt + # breakdown that was excluded from step_ms above. + self.tb_logger.add_scalar( + "perf/train_wall_step_time_ms", wall_step_ms, global_step=step + ) + self.tb_logger.add_scalar( + "perf/window_eval_time_ms", eval_ms, global_step=step + ) + self.tb_logger.add_scalar( + "perf/window_ckpt_time_ms", ckpt_ms, global_step=step + ) + self.tb_logger.add_scalar( + "perf/train_total_samples", self._perf_total_samples, global_step=step + ) + self.tb_logger.add_scalar( + "perf/train_elapsed_sec", elapsed, global_step=step + ) + # TFLOPS / MFU reporting (algo = dense yardstick, real = jagged). + # tflops_algo/gpu, mfu — uses max_seq_len^2 attention work (the + # MFU yardstick: the FLOPs the workload would do if every + # user's UIH filled the padded seq length). + # tflops_real/gpu, hfu — uses this batch's mean(s_i^2) (actual + # GPU work; hardware utilization). + # fill — real / algo as a percent; how much of + # the algo budget the model actually executed this batch. + # The jagged stash is read from the inner model; the model ref may + # be a DMP wrapper, so unwrap via .module if present. + tflops_str = "" + if self._num_flops_per_sample > 0 and self._gpu_peak_flops > 0: + local_flops = self._num_flops_per_sample * local_sps + tflops_algo = local_flops / 1e12 + mfu = 100.0 * local_flops / self._gpu_peak_flops + self.tb_logger.add_scalar("perf/train_tflops_algo_gpu", tflops_algo, global_step=step) + self.tb_logger.add_scalar("perf/train_mfu_pct", mfu, global_step=step) + tflops_str = f" tflops_algo/gpu={tflops_algo:.1f} mfu={mfu:.1f}%" + jagged_t = None + m = self._model_ref + if m is not None: + inner = m.module if hasattr(m, "module") else m + jagged_t = getattr(inner, "_last_jagged_flops_per_sample", None) + if jagged_t is not None: + jagged = float(jagged_t.item()) + if 0 < jagged < self._num_flops_per_sample: + tflops_real = jagged * local_sps / 1e12 + hfu = 100.0 * jagged * local_sps / self._gpu_peak_flops + fill = 100.0 * jagged / self._num_flops_per_sample + self.tb_logger.add_scalar("perf/train_tflops_real_gpu", tflops_real, global_step=step) + self.tb_logger.add_scalar("perf/train_hfu_pct", hfu, global_step=step) + self.tb_logger.add_scalar("perf/train_fill_pct", fill, global_step=step) + tflops_str += f" tflops_real/gpu={tflops_real:.1f} hfu={hfu:.1f}% fill={fill:.1f}%" + logger.info( + f"train - Step {step} perf: local_sps={local_sps:.1f} " + f"global_sps={global_sps:.1f} step_ms={step_ms:.2f} " + f"elapsed_sec={elapsed:.1f} total_samples={self._perf_total_samples} " + f"wall_step_ms={wall_step_ms:.2f} eval_ms={eval_ms:.1f} " + f"ckpt_ms={ckpt_ms:.1f}" + + tflops_str + ) + self._perf_t_window = now + self._perf_steps_in_window = 0 + self._perf_samples_counter.zero_() + # Reset the excluded-time accumulators for the next window. Any + # still-open pause (eval/ckpt straddling this log) is cleared so its + # remaining time is not double-counted; in practice perf logs fire + # only after a train step, never mid eval/ckpt. + self._perf_excluded = {"eval": 0.0, "ckpt": 0.0} + self._perf_pause = {} + + # Time-to-target: latch wall-clock once any task's AUC crosses threshold. + # Matches MLPerf DLRM-DCNv2 reporting style (default upstream target 0.80275). + if ( + self._auc_threshold is not None + and not self._time_to_target_logged + ): + for key, val in all_computed_metrics.items(): + metric_short = key.split("/")[-2] if "/" in key else key + if metric_short.endswith("auc") and not metric_short.endswith("gauc"): + if float(val) >= self._auc_threshold: + ttt = time.perf_counter() - self._perf_t_start + self.tb_logger.add_scalar( + f"perf/time_to_auc_{self._auc_threshold:.5f}_sec", + ttt, + global_step=self.global_step[mode], + ) + logger.info( + f"REACHED AUC>={self._auc_threshold} on {key}=" + f"{float(val):.6f} at elapsed_sec={ttt:.2f} " + f"step={self.global_step[mode]}" + ) + self._time_to_target_logged = True + break + + return all_computed_metrics + + def reset(self, mode: str = "train"): + """ + Reset all metrics for a given mode. + + Args: + mode: Either 'train' or 'eval'. + """ + for metric in self.all_metrics[mode]: + metric.reset() + + +# the datasets we support +SUPPORTED_DATASETS = [ + "debug", + "movielens-1m", + "movielens-20m", + "movielens-13b", + "movielens-18b", + "kuairand-1k", + "streaming-400m", + "streaming-200b", + "streaming-100b", + "sampled-streaming-100b", + "yambda-5b", +] + + +@gin.configurable +def env_path(key: str = "", default: str = "") -> str: + """Resolve a path from os.environ[key], falling back to `default`. + + Intended as a gin macro so paths can be overridden via env vars without + editing the gin file. Example gin usage: + + DATA_PATH = @env_path() + env_path.key = "DLRM_DATA_PATH" + env_path.default = "/some/default/path" + make_train_test_dataloaders.new_path_prefix = %DATA_PATH + """ + return os.environ.get(key, default) if key else default + + +@gin.configurable +def env_str(key: str = "", default: str = "") -> str: + """Resolve a string from os.environ[key], falling back to `default`. + + Companion to `env_int`/`env_float` for categorical/string overrides (e.g. a + metric backend selector). Example gin usage: + + MetricsLogger.train_lifetime_auc_mode = @tlam/env_str() + tlam/env_str.key = "TRAIN_LIFETIME_AUC_MODE" + tlam/env_str.default = "binned" + """ + raw = os.environ.get(key) if key else None + return raw if raw else default + + +@gin.configurable +def env_str_map( + key: str = "", + default: Optional[Dict[str, str]] = None, + merge: bool = False, +) -> Dict[str, str]: + """Parse os.environ[key] as 'name=value,name=value' into a dict. + + Falls back to `default` (gin) when the env var is unset/empty. Companion to + `env_str` for map-valued overrides (e.g. per-table embedding placement). + Example gin usage: + + make_optimizer_and_shard.embedding_placement_overrides = @env_str_map() + env_str_map.key = "EMB_PLACEMENT_OVERRIDES" + env_str_map.default = {} + + Example env override: EMB_PLACEMENT_OVERRIDES="uid=uvm_caching,item_id=hbm". + + `merge` controls how the parsed env entries combine with `default`: + * False (default): the env var REPLACES `default` wholesale (whole-dict + override) — set the env var and you fully define the map. + * True: the parsed env entries are LAYERED ON TOP of `default` (per-key + override) — gin `default` defines the base map and the env var tweaks + only the named keys, leaving the rest of `default` intact. + """ + base = dict(default or {}) + raw = os.environ.get(key) if key else None + if not raw: + return base + parsed: Dict[str, str] = {} + for pair in raw.split(","): + pair = pair.strip() + if not pair: + continue + k, _, v = pair.partition("=") + parsed[k.strip()] = v.strip() + return {**base, **parsed} if merge else parsed + + +@gin.configurable +def env_int(key: str = "", default: int = 0) -> int: + """Resolve an int from os.environ[key], falling back to `default`. + + Companion to `env_path` for numeric overrides. Example gin usage: + + make_optimizer_and_shard.hbm_cap_gb = @env_int() + env_int.key = "HBM_CAP_GB" + env_int.default = 260 + """ + raw = os.environ.get(key) if key else None + return int(raw) if raw else default + + +@gin.configurable +def env_float(key: str = "", default: float = 0.0) -> float: + """Resolve a float from os.environ[key], falling back to `default`. + + Companion to `env_int` for fractional/duration overrides (e.g. a + checkpoint time interval in seconds). Example gin usage: + + streaming_train_eval_loop.checkpoint_time_interval_s = @env_float() + env_float.key = "CKPT_TIME_INTERVAL_S" + env_float.default = 3600.0 + """ + raw = os.environ.get(key) if key else None + return float(raw) if raw else default + + +_GPU_PEAK_FLOPS_TABLE: Dict[str, Dict[str, float]] = { + # Per-GPU peak TFLOPS by dtype. Values from vendor datasheets / Primus-DLRM + # peak_table. Used as the denominator in MFU/HFU. Keyed by case-insensitive + # substring of torch.cuda.get_device_name(0). + "MI355X": {"bf16": 2300e12, "fp32": 575e12}, + "MI350X": {"bf16": 2300e12, "fp32": 575e12}, + "MI300X": {"bf16": 1300e12, "fp32": 653e12}, + "MI325X": {"bf16": 1300e12, "fp32": 653e12}, + "B200": {"bf16": 2250e12, "fp32": 1125e12}, + "H100": {"bf16": 990e12, "fp32": 67e12}, + "A100": {"bf16": 312e12, "fp32": 19.5e12}, +} + + +def get_gpu_peak_flops(dtype: str = "bf16") -> float: + """Peak FLOPS for the current GPU at the given dtype. + + Falls back to MI350X's number with a warning when the device name doesn't + match any table entry — better to over-report MFU than to silently skip. + """ + if not torch.cuda.is_available(): + return 0.0 + name = torch.cuda.get_device_name(0) + for gpu_key, peaks in _GPU_PEAK_FLOPS_TABLE.items(): + if gpu_key in name: + return peaks.get(dtype, peaks["bf16"]) + logger.warning( + f"Unknown GPU for peak FLOPS: {name}; defaulting to MI350X bf16 (2300 TF)" + ) + return _GPU_PEAK_FLOPS_TABLE["MI350X"]["bf16"] + + +@gin.configurable +def run_results_dir(run_name: str = "default", subdir: str = "results") -> str: + """Resolve ``//`` from this file's location. + + Used as a gin macro to give per-run output directories that persist on the + host (recommendation_v4 is bind-mounted into the training container). + + Example gin usage:: + + RUN_NAME = @env_path() + env_path.key = "RUN_NAME" + env_path.default = "default" + run_results_dir.run_name = %RUN_NAME + Profiler.trace_dir = @run_results_dir() + """ + # utils.py lives at /generative_recommenders/dlrm_v3/utils.py; + # parents[2] climbs to /. + repo_root = Path(__file__).resolve().parents[2] + return str(repo_root / subdir / run_name) + + +@gin.configurable +def get_dataset( + name: str, + new_path_prefix: str = "", + history_length: Optional[int] = None, + min_history: Optional[int] = None, + history_strategy: str = "interleaved", + streaming_window_seconds: int = 86400, + streaming_sort_within_window: bool = False, + streaming_shuffle_fraction: float = 0.0, + streaming_shuffle_seed: int = 0, + train_split_percentage: float = 1.0, + split_salt: int = 0, +): + """ + Get dataset class and configuration by name. + + Args: + name: Dataset identifier (must be in SUPPORTED_DATASETS). + new_path_prefix: Optional prefix to prepend to data paths. + + Returns: + Tuple of (dataset_class, kwargs_dict) for dataset instantiation. + + Raises: + AssertionError: If dataset name is not supported. + """ + assert name in SUPPORTED_DATASETS, f"dataset {name} not supported" + if name == "debug": + return DLRMv3RandomDataset, {} + if name == "movielens-1m": + return ( + DLRMv3MovieLensDataset, + { + "ratings_file": os.path.join( + new_path_prefix, "data/ml-1m/sasrec_format.csv" + ), + }, + ) + if name == "movielens-20m": + return ( + DLRMv3MovieLensDataset, + { + "ratings_file": os.path.join( + new_path_prefix, "data/ml-20m/sasrec_format.csv" + ), + }, + ) + if name == "movielens-13b": + return ( + DLRMv3SyntheticMovieLensDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/ml-13b/16x16384" + ), + }, + ) + if name == "movielens-18b": + return ( + DLRMv3SyntheticMovieLensDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/ml-18b/20x36864" + ), + }, + ) + if name == "kuairand-1k": + return ( + DLRMv3KuaiRandDataset, + { + "seq_logs_file": os.path.join( + new_path_prefix, "data/KuaiRand-1K/data/processed_seqs.csv" + ), + }, + ) + if name == "streaming-400m": + return ( + DLRMv3SyntheticStreamingDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/streaming-400m/" + ), + "train_ts": 8, + "total_ts": 10, + "num_files": 3, + "num_users": 150_000, + "num_items": 1_500_000, + "num_categories": 128, + }, + ) + if name == "streaming-200b": + return ( + DLRMv3SyntheticStreamingDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/streaming-200b/" + ), + "train_ts": 90, + "total_ts": 100, + "num_files": 100, + "num_users": 10_000_000, + "num_items": 1_000_000_000, + "num_categories": 128, + }, + ) + if name == "streaming-100b": + return ( + DLRMv3SyntheticStreamingDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/streaming-100b/" + ), + "train_ts": 90, + "total_ts": 100, + "num_files": 100, + "num_users": 5_000_000, + "num_items": 1_000_000_000, + "num_categories": 128, + }, + ) + if name == "yambda-5b": + from generative_recommenders.dlrm_v3.configs import YAMBDA_5B_CROSS_SPECS + + return ( + DLRMv3YambdaDataset, + { + # Layout: /processed_5b/{train_sessions.parquet,...} + # and /shared_metadata/{artist,album}_item_mapping.parquet. + # The dataset auto-builds a MAP_SHARED-mmap'd cache of the + # flat columns + LISTEN-anchor positions under + # /hstu_cache_L/ on first use; + # all ranks on a node share the same physical pages. + "processed_dir": os.path.join(new_path_prefix, "processed_5b"), + "metadata_dir": os.path.join(new_path_prefix, "shared_metadata"), + # Per-pool truncation cap; total interleaved UIH ~ 3*L/3 = L. + # Override via `get_dataset.history_length = N` in gin. + "history_length": history_length if history_length is not None else 4096, + "scan_window": 20000, + # Anchor-eligibility floor: a LISTEN event qualifies once the + # user has >= min_history prior events. Decoupled from + # history_length (gather cap) since jagged attention handles + # short UIH. None = legacy (require a full history_length). + # Override via `get_dataset.min_history = N` / $MIN_HISTORY. + "min_history": min_history, + # UIH construction: "interleaved" (per-pool L//3 cap) or + # "last_n" (last history_length pooled events, no per-pool + # split). Strategy-independent on disk — both reuse the same + # hstu_cache_L/ and positions file (the gather + # runs at sample-construction time), so switching needs no + # rebuild. Override via $HISTORY_STRATEGY. + "history_strategy": history_strategy, + "cross_specs": YAMBDA_5B_CROSS_SPECS, + # Temporal-streaming knobs (only used under --mode + # streaming-train-eval; ignored by the default train-eval path). + "streaming_window_seconds": streaming_window_seconds, + "streaming_sort_within_window": streaming_sort_within_window, + # In-window shuffle diversity dial in [0,1]: K=round(frac*N) within- + # segment shuffle. 0=off/user-major, 1=full. Config-invariant and + # deterministic by (seed, ts). + "streaming_shuffle_fraction": streaming_shuffle_fraction, + "streaming_shuffle_seed": streaming_shuffle_seed, + # User-level train:eval holdout for the streaming path. 1.0 = + # no holdout (legacy). <1.0 holds out (1 - tsp) of users as a + # fixed eval set; those users are never trained. + "train_split_percentage": train_split_percentage, + "split_salt": split_salt, + }, + ) + if name == "sampled-streaming-100b": + return ( + DLRMv3SyntheticStreamingDataset, + { + "ratings_file_prefix": os.path.join( + new_path_prefix, "data/streaming-100b/sampled_data/" + ), + "train_ts": 90, + "total_ts": 100, + "num_files": 1, + "num_users": 50_000, + "num_items": 1_000_000_000, + "num_categories": 128, + }, + ) diff --git a/recommendation_v4/generative_recommenders/modules/action_encoder.py b/recommendation_v4/generative_recommenders/modules/action_encoder.py new file mode 100644 index 000000000..13b65557e --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/action_encoder.py @@ -0,0 +1,112 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Dict, List, Optional, Tuple + +import torch +from generative_recommenders.common import HammerModule +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged + + +class ActionEncoder(HammerModule): + def __init__( + self, + action_embedding_dim: int, + action_feature_name: str, + action_weights: List[int], + watchtime_feature_name: str = "", + watchtime_to_action_thresholds_and_weights: Optional[ + List[Tuple[int, int]] + ] = None, + embedding_init_std: float = 0.1, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._watchtime_feature_name: str = watchtime_feature_name + self._action_feature_name: str = action_feature_name + self._watchtime_to_action_thresholds_and_weights: List[Tuple[int, int]] = ( + watchtime_to_action_thresholds_and_weights + if watchtime_to_action_thresholds_and_weights is not None + else [] + ) + self.register_buffer( + "_combined_action_weights", + torch.tensor( + action_weights + + [x[1] for x in self._watchtime_to_action_thresholds_and_weights] + ), + ) + self._num_action_types: int = len(action_weights) + len( + self._watchtime_to_action_thresholds_and_weights + ) + self._action_embedding_dim = action_embedding_dim + self._action_embedding_table: torch.nn.Parameter = torch.nn.Parameter( + torch.empty((self._num_action_types, action_embedding_dim)).normal_( + mean=0, std=embedding_init_std + ), + ) + self._target_action_embedding_table: torch.nn.Parameter = torch.nn.Parameter( + torch.empty((1, self._num_action_types * action_embedding_dim)).normal_( + mean=0, std=embedding_init_std + ), + ) + + @property + def output_embedding_dim(self) -> int: + return self._action_embedding_dim * self._num_action_types + + def forward( + self, + max_uih_len: int, + max_targets: int, + uih_offsets: torch.Tensor, + target_offsets: torch.Tensor, + seq_embeddings: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + seq_actions = seq_payloads[self._action_feature_name] + if len(self._watchtime_to_action_thresholds_and_weights) > 0: + watchtimes = seq_payloads[self._watchtime_feature_name] + for threshold, weight in self._watchtime_to_action_thresholds_and_weights: + seq_actions = torch.bitwise_or( + seq_actions, (watchtimes >= threshold).to(torch.int64) * weight + ) + exploded_actions = ( + torch.bitwise_and( + seq_actions.unsqueeze(-1), self._combined_action_weights.unsqueeze(0) + ) + > 0 + ) + action_embeddings = ( + exploded_actions.unsqueeze(-1) * self._action_embedding_table.unsqueeze(0) + ).view(-1, self._num_action_types * self._action_embedding_dim) + total_targets: int = seq_embeddings.size(0) - action_embeddings.size(0) + action_embeddings = concat_2D_jagged( + max_seq_len=max_uih_len + max_targets, + values_left=action_embeddings, + values_right=self._target_action_embedding_table.tile( + total_targets, + 1, + ), + max_len_left=max_uih_len, + max_len_right=max_targets, + offsets_left=uih_offsets, + offsets_right=target_offsets, + kernel=self.hammer_kernel(), + ) + return action_embeddings diff --git a/recommendation_v4/generative_recommenders/modules/content_encoder.py b/recommendation_v4/generative_recommenders/modules/content_encoder.py new file mode 100644 index 000000000..acca82dbf --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/content_encoder.py @@ -0,0 +1,109 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Dict, List, Optional + +import torch +from generative_recommenders.common import HammerModule +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged + + +class ContentEncoder(HammerModule): + def __init__( + self, + input_embedding_dim: int, + additional_content_features: Optional[Dict[str, int]] = None, + target_enrich_features: Optional[Dict[str, int]] = None, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._input_embedding_dim: int = input_embedding_dim + self._additional_content_features: Dict[str, int] = ( + additional_content_features + if additional_content_features is not None + else {} + ) + self._target_enrich_features: Dict[str, int] = ( + target_enrich_features if target_enrich_features is not None else {} + ) + self._target_enrich_dummy_embeddings: torch.nn.ParameterDict = ( + torch.nn.ParameterDict( + { + name: torch.nn.Parameter( + torch.empty((1, dim)).normal_(mean=0, std=0.1), + ) + for name, dim in self._target_enrich_features.items() + } + ) + ) + + @property + def output_embedding_dim(self) -> int: + return self._input_embedding_dim + sum( + list(self._additional_content_features.values()) + + list(self._target_enrich_features.values()) + ) + + def forward( + self, + max_uih_len: int, + max_targets: int, + uih_offsets: torch.Tensor, + target_offsets: torch.Tensor, + seq_embeddings: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + content_embeddings_list: List[torch.Tensor] = [seq_embeddings] + if len(self._additional_content_features) > 0: + content_embeddings_list = content_embeddings_list + [ + (seq_payloads[x].to(seq_embeddings.dtype)) + for x in self._additional_content_features.keys() + ] + + if self._target_enrich_dummy_embeddings: + total_seq_len: int = seq_embeddings.size(0) + for name, param in self._target_enrich_dummy_embeddings.items(): + enrich_embeddings_target = seq_payloads[name].to(seq_embeddings.dtype) + total_targets: int = enrich_embeddings_target.size(0) + total_uih_len: int = total_seq_len - total_targets + enrich_embeddings_uih = param.tile(total_uih_len, 1).to( + seq_embeddings.dtype + ) + enrich_embeddings = concat_2D_jagged( + max_seq_len=max_uih_len + max_targets, + values_left=enrich_embeddings_uih, + values_right=enrich_embeddings_target, + max_len_left=max_uih_len, + max_len_right=max_targets, + offsets_left=uih_offsets, + offsets_right=target_offsets, + kernel=self.hammer_kernel(), + ) + content_embeddings_list.append(enrich_embeddings) + + if ( + len(self._target_enrich_features) == 0 + and len(self._additional_content_features) == 0 + ): + return seq_embeddings + else: + content_embeddings = torch.cat( + content_embeddings_list, + dim=1, + ) + return content_embeddings diff --git a/recommendation_v4/generative_recommenders/modules/contextual_interleave_preprocessor.py b/recommendation_v4/generative_recommenders/modules/contextual_interleave_preprocessor.py new file mode 100644 index 000000000..fff0d72f0 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/contextual_interleave_preprocessor.py @@ -0,0 +1,357 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from math import sqrt +from typing import Callable, Dict, Optional, Tuple + +import torch +from generative_recommenders.common import fx_unwrap_optional_tensor +from generative_recommenders.modules.action_encoder import ActionEncoder +from generative_recommenders.modules.content_encoder import ContentEncoder +from generative_recommenders.modules.contextualize_mlps import ( + ContextualizedMLP, + ParameterizedContextualizedMLP, +) +from generative_recommenders.modules.preprocessors import ( + get_contextual_input_embeddings, + InputPreprocessor, +) +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged + + +class ContextualInterleavePreprocessor(InputPreprocessor): + def __init__( + self, + input_embedding_dim: int, + output_embedding_dim: int, + contextual_feature_to_max_length: Dict[str, int], + contextual_feature_to_min_uih_length: Dict[str, int], + content_encoder: ContentEncoder, + content_contextualize_mlp_fn: Callable[ + [int, int, int, bool], ContextualizedMLP + ], + action_encoder: ActionEncoder, + action_contextualize_mlp_fn: Callable[[int, int, int, bool], ContextualizedMLP], + pmlp_contextual_dropout_ratio: float = 0.0, + enable_interleaving: bool = False, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._input_embedding_dim: int = input_embedding_dim + self._output_embedding_dim: int = output_embedding_dim + self._contextual_feature_to_max_length: Dict[str, int] = ( + contextual_feature_to_max_length + ) + self._max_contextual_seq_len: int = sum( + contextual_feature_to_max_length.values() + ) + self._contextual_feature_to_min_uih_length: Dict[str, int] = ( + contextual_feature_to_min_uih_length + ) + std = 1.0 * sqrt(2.0 / float(input_embedding_dim + output_embedding_dim)) + self._batched_contextual_linear_weights = torch.nn.Parameter( + torch.empty( + ( + self._max_contextual_seq_len, + input_embedding_dim, + output_embedding_dim, + ) + ).normal_(0.0, std) + ) + self._pmlp_contextual_dropout_ratio: float = pmlp_contextual_dropout_ratio + self._batched_contextual_linear_bias = torch.nn.Parameter( + torch.empty((self._max_contextual_seq_len, 1, output_embedding_dim)).fill_( + 0.0 + ) + ) + contextual_embedding_dim: int = ( + self._max_contextual_seq_len * input_embedding_dim + ) + self._content_encoder: ContentEncoder = content_encoder + self._content_embedding_mlp: ContextualizedMLP = content_contextualize_mlp_fn( + self._content_encoder.output_embedding_dim, + output_embedding_dim, + contextual_embedding_dim, + is_inference, + ) + self._action_encoder: ActionEncoder = action_encoder + self._action_embedding_mlp: ContextualizedMLP = action_contextualize_mlp_fn( + self._action_encoder.output_embedding_dim, + output_embedding_dim, + contextual_embedding_dim, + is_inference, + ) + self._enable_interleaving: bool = enable_interleaving + + def combine_embeddings( + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + content_embeddings: torch.Tensor, + action_embeddings: torch.Tensor, + contextual_embeddings: Optional[torch.Tensor], + num_targets: torch.Tensor, + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + ]: + if self._enable_interleaving: + output_seq_timestamps = seq_timestamps.repeat_interleave(2) + output_seq_embeddings = torch.stack( + [content_embeddings, action_embeddings], dim=1 + ).reshape(-1, self._output_embedding_dim) + if self.interleave_targets(): + output_seq_lengths = seq_lengths * 2 + output_max_seq_len = (max_uih_len + max_targets) * 2 + output_num_targets = num_targets * 2 + output_total_uih_len = total_uih_len * 2 + output_total_targets = total_targets * 2 + else: + seq_lengths_by_2 = seq_lengths * 2 + output_seq_lengths = seq_lengths_by_2 - num_targets + output_max_seq_len = 2 * max_uih_len + max_targets + indices = torch.arange( + 2 * (max_uih_len + max_targets), device=seq_lengths.device + ).view(1, -1) + valid_mask = torch.logical_and( + indices < seq_lengths_by_2.view(-1, 1), + torch.logical_or( + indices < (output_seq_lengths - num_targets).view(-1, 1), + torch.remainder(indices, 2) == 0, + ), + ) + jagged_valid_mask = ( + torch.ops.fbgemm.dense_to_jagged( + valid_mask.int().unsqueeze(-1), + [ + torch.ops.fbgemm.asynchronous_complete_cumsum( + seq_lengths_by_2 + ) + ], + )[0] + .to(torch.bool) + .squeeze(1) + ) + output_seq_embeddings = output_seq_embeddings[jagged_valid_mask] + output_seq_timestamps = output_seq_timestamps[jagged_valid_mask] + output_num_targets = num_targets + output_total_uih_len = total_uih_len * 2 + output_total_targets = total_targets + else: + output_max_seq_len = max_uih_len + max_targets + output_seq_lengths = seq_lengths + output_num_targets = num_targets + output_seq_timestamps = seq_timestamps + output_seq_embeddings = content_embeddings + action_embeddings + output_total_uih_len = total_uih_len + output_total_targets = total_targets + + # concat contextual embeddings + output_seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + output_seq_lengths + ) + if self._max_contextual_seq_len > 0: + output_seq_embeddings = concat_2D_jagged( + max_seq_len=self._max_contextual_seq_len + output_max_seq_len, + values_left=fx_unwrap_optional_tensor(contextual_embeddings).reshape( + -1, self._output_embedding_dim + ), + values_right=output_seq_embeddings, + max_len_left=self._max_contextual_seq_len, + max_len_right=output_max_seq_len, + offsets_left=None, + offsets_right=output_seq_offsets, + kernel=self.hammer_kernel(), + ) + output_seq_timestamps = concat_2D_jagged( + max_seq_len=self._max_contextual_seq_len + output_max_seq_len, + values_left=torch.zeros( + (output_seq_lengths.size(0) * self._max_contextual_seq_len, 1), + dtype=output_seq_timestamps.dtype, + device=output_seq_timestamps.device, + ), + values_right=output_seq_timestamps.unsqueeze(-1), + max_len_left=self._max_contextual_seq_len, + max_len_right=output_max_seq_len, + offsets_left=None, + offsets_right=output_seq_offsets, + kernel=self.hammer_kernel(), + ).squeeze(-1) + output_max_seq_len = output_max_seq_len + self._max_contextual_seq_len + output_total_uih_len = ( + output_total_uih_len + + self._max_contextual_seq_len * output_seq_lengths.size(0) + ) + output_seq_lengths = output_seq_lengths + self._max_contextual_seq_len + output_seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + output_seq_lengths + ) + + return ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + ) + + def forward( # noqa C901 + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + ]: + max_seq_len = max_uih_len + max_targets + with torch.autocast( + "cuda", + dtype=torch.bfloat16, + enabled=(not self.is_inference and self._training_dtype == torch.bfloat16), + ): + # get contextual_embeddings + contextual_embeddings: Optional[torch.Tensor] = None + pmlp_contextual_embeddings: Optional[torch.Tensor] = None + if self._max_contextual_seq_len > 0: + contextual_input_embeddings = get_contextual_input_embeddings( + seq_lengths=seq_lengths, + seq_payloads=seq_payloads, + contextual_feature_to_max_length=self._contextual_feature_to_max_length, + contextual_feature_to_min_uih_length=self._contextual_feature_to_min_uih_length, + dtype=seq_embeddings.dtype, + ) + if isinstance( + self._action_embedding_mlp, ParameterizedContextualizedMLP + ) or isinstance( + self._action_embedding_mlp, ParameterizedContextualizedMLP + ): + pmlp_contextual_embeddings = torch.nn.functional.dropout( + contextual_input_embeddings, + p=self._pmlp_contextual_dropout_ratio, + training=self.training, + ) + contextual_embeddings = torch.baddbmm( + self._batched_contextual_linear_bias.to( + contextual_input_embeddings.dtype + ), + contextual_input_embeddings.view( + -1, self._max_contextual_seq_len, self._input_embedding_dim + ).transpose(0, 1), + self._batched_contextual_linear_weights.to( + contextual_input_embeddings.dtype + ), + ).transpose(0, 1) + + # content embeddings + seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(seq_lengths) + target_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(num_targets) + uih_offsets = seq_offsets - target_offsets + content_embeddings = self._content_encoder( + max_uih_len=max_uih_len, + max_targets=max_targets, + uih_offsets=uih_offsets, + target_offsets=target_offsets, + seq_embeddings=seq_embeddings, + seq_payloads=seq_payloads, + ) + content_embeddings = self._content_embedding_mlp( + seq_embeddings=content_embeddings, + seq_offsets=seq_offsets, + max_seq_len=max_seq_len, + contextual_embeddings=pmlp_contextual_embeddings, + ) + + # action embeddings + action_embeddings = self._action_encoder( + max_uih_len=max_uih_len, + max_targets=max_targets, + uih_offsets=uih_offsets, + target_offsets=target_offsets, + seq_embeddings=seq_embeddings, + seq_payloads=seq_payloads, + ).to(seq_embeddings.dtype) + action_embeddings = self._action_embedding_mlp( + seq_embeddings=action_embeddings, + seq_offsets=seq_offsets, + max_seq_len=max_seq_len, + contextual_embeddings=pmlp_contextual_embeddings, + ) + + ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + ) = self.combine_embeddings( + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_timestamps=seq_timestamps, + content_embeddings=content_embeddings, + action_embeddings=action_embeddings, + contextual_embeddings=contextual_embeddings, + num_targets=num_targets, + ) + + return ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + seq_payloads, + ) + + def interleave_targets(self) -> bool: + return self.is_train and self._enable_interleaving diff --git a/recommendation_v4/generative_recommenders/modules/contextualize_mlps.py b/recommendation_v4/generative_recommenders/modules/contextualize_mlps.py new file mode 100644 index 000000000..dc49effeb --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/contextualize_mlps.py @@ -0,0 +1,141 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict +import abc +from typing import Optional + +import torch +from generative_recommenders.common import HammerModule, init_mlp_weights_optional_bias +from generative_recommenders.ops.jagged_tensors import jagged_dense_bmm_broadcast_add +from generative_recommenders.ops.layer_norm import LayerNorm, SwishLayerNorm +from libfb.py.pyre import none_throws + + +class ContextualizedMLP(HammerModule): + @abc.abstractmethod + def forward( + self, + max_seq_len: int, + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + contextual_embeddings: Optional[torch.Tensor], + ) -> torch.Tensor: + """ + Args: + seq_embeddings: (L, D) + seq_offsets: (B + 1,) + max_seq_len: int + contextual_embeddings: (B, D') + """ + pass + + +class SimpleContextualizedMLP(ContextualizedMLP): + def __init__( + self, + sequential_input_dim: int, + sequential_output_dim: int, + hidden_dim: int, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=sequential_input_dim, + out_features=hidden_dim, + ), + SwishLayerNorm(hidden_dim, is_inference=is_inference), + torch.nn.Linear( + in_features=hidden_dim, + out_features=sequential_output_dim, + ), + LayerNorm(sequential_output_dim), + ).apply(init_mlp_weights_optional_bias) + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + max_seq_len: int, + contextual_embeddings: Optional[torch.Tensor], + ) -> torch.Tensor: + return self._mlp(seq_embeddings) + + +class ParameterizedContextualizedMLP(ContextualizedMLP): + def __init__( + self, + contextual_embedding_dim: int, + sequential_input_dim: int, + sequential_output_dim: int, + hidden_dim: int, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + + self._sequential_input_dim: int = sequential_input_dim + self._sequential_output_dim: int = sequential_output_dim + + self._dense_features_compress: torch.nn.Module = torch.nn.Linear( + in_features=contextual_embedding_dim, + out_features=hidden_dim, + ).apply(init_mlp_weights_optional_bias) + + self._attn_raw_weights: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=hidden_dim, + out_features=sequential_input_dim * sequential_output_dim, + ), + ).apply(init_mlp_weights_optional_bias) + + self._attn_weights_norm: torch.nn.Module = torch.nn.LayerNorm( + [sequential_input_dim, sequential_output_dim] + ) + + self._res_weights: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=hidden_dim, + out_features=hidden_dim, + ), + SwishLayerNorm(hidden_dim), + torch.nn.Linear( + in_features=hidden_dim, + out_features=sequential_output_dim, + ), + ).apply(init_mlp_weights_optional_bias) + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + max_seq_len: int, + contextual_embeddings: Optional[torch.Tensor], + ) -> torch.Tensor: + shared_input = self._dense_features_compress(none_throws(contextual_embeddings)) + attn_weights = self._attn_weights_norm( + self._attn_raw_weights(shared_input).reshape( + -1, self._sequential_input_dim, self._sequential_output_dim + ) + ) + return jagged_dense_bmm_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=seq_embeddings, + dense=attn_weights.to(seq_embeddings.dtype), + bias=self._res_weights(shared_input), + kernel=self.hammer_kernel(), + ) diff --git a/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py b/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py new file mode 100644 index 000000000..f11bb226a --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/dlrm_hstu.py @@ -0,0 +1,757 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + + +import logging +from dataclasses import dataclass, field +from typing import Dict, List, NamedTuple, Optional, Tuple + +import torch +from generative_recommenders.common import ( + fx_infer_max_len, + fx_mark_length_features, + HammerKernel, + HammerModule, + init_mlp_weights_optional_bias, + set_static_max_seq_lens, +) +from generative_recommenders.modules.hstu_transducer import HSTUTransducer +from generative_recommenders.modules.multitask_module import ( + DefaultMultitaskModule, + MultitaskTaskType, + TaskConfig, +) +from generative_recommenders.modules.positional_encoder import HSTUPositionalEncoder +from generative_recommenders.modules.postprocessors import ( + LayerNormPostprocessor, + TimestampLayerNormPostprocessor, +) +from generative_recommenders.modules.preprocessors import ContextualPreprocessor +from generative_recommenders.modules.stu import STU, STULayer, STULayerConfig, STUStack +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged +from generative_recommenders.ops.layer_norm import LayerNorm, SwishLayerNorm +from torch.autograd.profiler import record_function +from torchrec import KeyedJaggedTensor +from torchrec.modules.embedding_configs import EmbeddingConfig +from torchrec.modules.embedding_modules import EmbeddingCollection + +logger: logging.Logger = logging.getLogger(__name__) + +def fx_total_targets(num_candidates: torch.Tensor) -> int: + """Sum a per-sample candidate-count tensor to a Python int. + + Wrapped with ``torch.fx.wrap`` so ``TrainPipelineSparseDist``'s symbolic + trace treats it as an opaque leaf instead of recursing into the data- + dependent ``int(Proxy.sum().item())`` (which raises during tracing). + """ + return int(num_candidates.sum().item()) + + +torch.fx.wrap("fx_infer_max_len") +torch.fx.wrap("fx_total_targets") +torch.fx.wrap("len") + + +class SequenceEmbedding(NamedTuple): + lengths: torch.Tensor + embedding: torch.Tensor + + +@dataclass +class DlrmHSTUConfig: + max_seq_len: int = 16384 + max_num_candidates: int = 10 + max_num_candidates_inference: int = 5 + hstu_num_heads: int = 1 + hstu_attn_linear_dim: int = 256 + hstu_attn_qk_dim: int = 128 + hstu_attn_num_layers: int = 12 + hstu_embedding_table_dim: int = 192 + hstu_preprocessor_hidden_dim: int = 256 + hstu_transducer_embedding_dim: int = 0 + hstu_group_norm: bool = False + hstu_input_dropout_ratio: float = 0.2 + hstu_linear_dropout_rate: float = 0.2 + contextual_feature_to_max_length: Dict[str, int] = field(default_factory=dict) + contextual_feature_to_min_uih_length: Dict[str, int] = field(default_factory=dict) + candidates_weight_feature_name: str = "" + candidates_watchtime_feature_name: str = "" + candidates_querytime_feature_name: str = "" + causal_multitask_weights: float = 0.2 + multitask_configs: List[TaskConfig] = field(default_factory=list) + user_embedding_feature_names: List[str] = field(default_factory=list) + item_embedding_feature_names: List[str] = field(default_factory=list) + uih_post_id_feature_name: str = "" + uih_action_time_feature_name: str = "" + uih_weight_feature_name: str = "" + hstu_uih_feature_names: List[str] = field(default_factory=list) + hstu_candidate_feature_names: List[str] = field(default_factory=list) + merge_uih_candidate_feature_mapping: List[Tuple[str, str]] = field( + default_factory=list + ) + action_weights: Optional[List[int]] = None + action_embedding_init_std: float = 0.1 + enable_postprocessor: bool = True + use_layer_norm_postprocessor: bool = False + + +def _get_supervision_labels_and_weights( + supervision_bitmasks: torch.Tensor, + watchtime_sequence: torch.Tensor, + task_configs: List[TaskConfig], +) -> Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]: + supervision_labels: Dict[str, torch.Tensor] = {} + supervision_weights: Dict[str, torch.Tensor] = {} + for task in task_configs: + if task.task_type == MultitaskTaskType.REGRESSION: + supervision_labels[task.task_name] = watchtime_sequence.to(torch.float32) + elif task.task_type == MultitaskTaskType.BINARY_CLASSIFICATION: + supervision_labels[task.task_name] = ( + torch.bitwise_and(supervision_bitmasks, task.task_weight) > 0 + ).to(torch.float32) + else: + raise RuntimeError("Unsupported MultitaskTaskType") + return supervision_labels, supervision_weights + + +class DlrmHSTU(HammerModule): + def __init__( # noqa C901 + self, + hstu_configs: DlrmHSTUConfig, + embedding_tables: Dict[str, EmbeddingConfig], + is_inference: bool, + is_dense: bool = False, + bf16_training: bool = True, + ) -> None: + super().__init__(is_inference=is_inference) + logger.info(f"Initialize HSTU module with configs {hstu_configs}") + # When True, forward() takes the whole `Samples` batch as its single + # positional arg and reads the pre-merged sparse KJT off it. This keeps + # the EmbeddingCollection's input a plain getattr on the batch placeholder + # so TorchRec's TrainPipelineSparseDist can pipeline its input_dist. Set + # by build_train_pipeline(); leave False for eager / inference / eval. + self._pipeline_mode: bool = False + self._hstu_configs = hstu_configs + self._bf16_training: bool = bf16_training + # Last batch's jagged FLOPs/sample (0-d tensor on GPU). Populated by + # main_forward; MetricsLogger reads + .item()s on each compute_and_log + # to compute tflops_real/gpu and hfu (vs dense yardstick from + # get_num_flops_per_sample()). + self._last_jagged_flops_per_sample: Optional[torch.Tensor] = None + set_static_max_seq_lens([self._hstu_configs.max_seq_len]) + + if not is_dense: + self._embedding_collection: EmbeddingCollection = EmbeddingCollection( + tables=list(embedding_tables.values()), + need_indices=False, + device=torch.device("meta"), + ) + + # multitask configs must be sorted by task types + self._multitask_configs: List[TaskConfig] = hstu_configs.multitask_configs + self._multitask_module = DefaultMultitaskModule( + task_configs=self._multitask_configs, + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + prediction_fn=lambda in_dim, num_tasks: torch.nn.Sequential( + torch.nn.Linear(in_features=in_dim, out_features=512), + SwishLayerNorm(512), + torch.nn.Linear(in_features=512, out_features=num_tasks), + ).apply(init_mlp_weights_optional_bias), + causal_multitask_weights=hstu_configs.causal_multitask_weights, + is_inference=self._is_inference, + ) + self._additional_embedding_features: List[str] = [ + uih_feature_name + for ( + uih_feature_name, + candidate_feature_name, + ) in self._hstu_configs.merge_uih_candidate_feature_mapping + if ( + candidate_feature_name + in self._hstu_configs.item_embedding_feature_names + ) + and (uih_feature_name in self._hstu_configs.user_embedding_feature_names) + and (uih_feature_name is not self._hstu_configs.uih_post_id_feature_name) + ] + + # preprocessor setup + preprocessor = ContextualPreprocessor( + input_embedding_dim=hstu_configs.hstu_embedding_table_dim, + hidden_dim=hstu_configs.hstu_preprocessor_hidden_dim, + output_embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + contextual_feature_to_max_length=hstu_configs.contextual_feature_to_max_length, + contextual_feature_to_min_uih_length=hstu_configs.contextual_feature_to_min_uih_length, + action_embedding_dim=8, + action_feature_name=self._hstu_configs.uih_weight_feature_name, + action_weights=self._hstu_configs.action_weights, + action_embedding_init_std=self._hstu_configs.action_embedding_init_std, + additional_embedding_features=self._additional_embedding_features, + is_inference=is_inference, + ) + + # positional encoder + positional_encoder = HSTUPositionalEncoder( + num_position_buckets=8192, + num_time_buckets=2048, + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + contextual_seq_len=sum( + dict(hstu_configs.contextual_feature_to_max_length).values() + ), + is_inference=self._is_inference, + ) + + if hstu_configs.enable_postprocessor: + if hstu_configs.use_layer_norm_postprocessor: + postprocessor = LayerNormPostprocessor( + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + eps=1e-5, + is_inference=self._is_inference, + ) + else: + postprocessor = TimestampLayerNormPostprocessor( + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + time_duration_features=[ + (60 * 60, 24), # hour of day + (24 * 60 * 60, 7), # day of week + # (24 * 60 * 60, 365), # time of year (approximate) + ], + eps=1e-5, + is_inference=self._is_inference, + ) + else: + postprocessor = None + + # construct HSTU + stu_module: STU = STUStack( + stu_list=[ + STULayer( + config=STULayerConfig( + embedding_dim=hstu_configs.hstu_transducer_embedding_dim, + num_heads=hstu_configs.hstu_num_heads, + hidden_dim=hstu_configs.hstu_attn_linear_dim, + attention_dim=hstu_configs.hstu_attn_qk_dim, + output_dropout_ratio=hstu_configs.hstu_linear_dropout_rate, + use_group_norm=hstu_configs.hstu_group_norm, + causal=True, + target_aware=True, + max_attn_len=None, + attn_alpha=None, + recompute_normed_x=True, + recompute_uvqk=True, + recompute_y=True, + sort_by_length=True, + contextual_seq_len=0, + ), + is_inference=is_inference, + ) + for _ in range(hstu_configs.hstu_attn_num_layers) + ], + is_inference=is_inference, + ) + self._hstu_transducer: HSTUTransducer = HSTUTransducer( + stu_module=stu_module, + input_preprocessor=preprocessor, + output_postprocessor=postprocessor, + input_dropout_ratio=hstu_configs.hstu_input_dropout_ratio, + positional_encoder=positional_encoder, + is_inference=self._is_inference, + return_full_embeddings=False, + listwise=False, + ) + + # item embeddings + self._item_embedding_mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=hstu_configs.hstu_embedding_table_dim + * len(self._hstu_configs.item_embedding_feature_names), + out_features=512, + ), + SwishLayerNorm(512), + torch.nn.Linear( + in_features=512, + out_features=hstu_configs.hstu_transducer_embedding_dim, + ), + LayerNorm(hstu_configs.hstu_transducer_embedding_dim), + ).apply(init_mlp_weights_optional_bias) + + # -- FLOPs estimation ----------------------------------------------------- + # Convention matches TorchTitan / Primus-DLRM: matmul = 6 × M × N × K + # (×3 fwd+bwd, ×2 FMA), attention = 2 matmuls (Q·K^T + att·V). + # Embedding lookups excluded — they're memory-bound, not compute. + # + # HSTU vs OneTrans: HSTU collapses attention + FFN into a single UVQK + # projection plus SiLU(U) ⊙ y elementwise gating. There is NO separate + # FFN block (which dominates FLOPs in a standard transformer), so HSTU + # is intentionally compute-leaner per layer for the same N. + def _hstu_layer_flops( + self, n_tokens_linear: float, n_tokens_attn_sq: float + ) -> float: + """Per-layer FLOPs given linear-op token count and attention-token² + count. Dense estimate uses ``N`` and ``N²``; jagged estimate + substitutes ``mean(s_i)`` and ``mean(s_i²)``.""" + cfg = self._hstu_configs + D = cfg.hstu_embedding_table_dim + H = cfg.hstu_num_heads + hd = cfg.hstu_attn_linear_dim # V/U head dim + qd = cfg.hstu_attn_qk_dim # Q/K head dim + uvqk = 6 * n_tokens_linear * D * (2 * hd + 2 * qd) * H + attn = 6 * n_tokens_attn_sq * H * (qd + hd) # Q·K^T + att·V + out = 6 * n_tokens_linear * (3 * H * hd) * D + return uvqk + attn + out + + def get_num_flops_per_sample(self) -> float: + """Dense-equivalent fwd+bwd FLOPs per sample at ``max_seq_len``. + + Used as the MFU yardstick (peak utilization the workload could + theoretically reach if every sample's sequence were the full padded + length). The actual ``tflops_real``/``hfu`` reported per step uses + the jagged estimate stashed by ``main_forward``. + """ + cfg = self._hstu_configs + N = float(cfg.max_seq_len) + n_layers = cfg.hstu_attn_num_layers + flops = n_layers * self._hstu_layer_flops( + n_tokens_linear=N, n_tokens_attn_sq=N * N + ) + # Multitask head (Linear(D, n_tasks)) — negligible but cheap to add. + n_tasks = len(self._multitask_configs) + if n_tasks > 0: + flops += 6 * n_tasks * cfg.hstu_embedding_table_dim + return float(flops) + + def _compute_jagged_flops_per_sample( + self, + uih_seq_lengths: torch.Tensor, + num_candidates: torch.Tensor, + ) -> torch.Tensor: + """Jagged fwd+bwd FLOPs per sample for THIS batch's actual lengths. + + Per-sample merged sequence length s_i = uih_seq_lengths[i] + + num_candidates[i]. Returns a 0-d tensor on the batch's device; + caller should ``.item()`` it (one D→H sync per logging interval). + """ + s = (uih_seq_lengths + num_candidates).float() + mean_s = s.mean() + mean_s_sq = (s * s).mean() + cfg = self._hstu_configs + n_layers = cfg.hstu_attn_num_layers + flops = n_layers * ( + 6 * mean_s * cfg.hstu_embedding_table_dim + * (2 * cfg.hstu_attn_linear_dim + 2 * cfg.hstu_attn_qk_dim) + * cfg.hstu_num_heads + + 6 * mean_s_sq * cfg.hstu_num_heads + * (cfg.hstu_attn_qk_dim + cfg.hstu_attn_linear_dim) + + 6 * mean_s * (3 * cfg.hstu_num_heads * cfg.hstu_attn_linear_dim) + * cfg.hstu_embedding_table_dim + ) + n_tasks = len(self._multitask_configs) + if n_tasks > 0: + flops = flops + 6 * n_tasks * cfg.hstu_embedding_table_dim + return flops + + def _construct_payload( + self, + payload_features: Dict[str, torch.Tensor], + seq_embeddings: Dict[str, SequenceEmbedding], + ) -> Dict[str, torch.Tensor]: + if len(self._hstu_configs.contextual_feature_to_max_length) > 0: + contextual_offsets: List[torch.Tensor] = [] + for x in self._hstu_configs.contextual_feature_to_max_length.keys(): + contextual_offsets.append( + torch.ops.fbgemm.asynchronous_complete_cumsum( + seq_embeddings[x].lengths + ) + ) + else: + # Dummy, offsets are unused + contextual_offsets = torch.empty((0, 0)) + if torch.jit.is_scripting(): + # Explicit loops are TS-clean (avoid the dict-merge / dict-comp + # idioms below, which TorchScript cannot script). + out: Dict[str, torch.Tensor] = {} + for k, v in payload_features.items(): + out[k] = v + for x in self._hstu_configs.contextual_feature_to_max_length.keys(): + out[x] = seq_embeddings[x].embedding + i = 0 + for x in self._hstu_configs.contextual_feature_to_max_length.keys(): + # pyre-ignore[6] + out[x + "_offsets"] = contextual_offsets[i] + i += 1 + for x in self._additional_embedding_features: + out[x] = seq_embeddings[x].embedding + return out + return { + **payload_features, + **{ + x: seq_embeddings[x].embedding + for x in self._hstu_configs.contextual_feature_to_max_length.keys() + }, + **{ + x + "_offsets": contextual_offsets[i] + for i, x in enumerate( + list(self._hstu_configs.contextual_feature_to_max_length.keys()) + ) + }, + **{ + x: seq_embeddings[x].embedding + for x in self._additional_embedding_features + }, + } + + def _user_forward( + self, + max_uih_len: int, + max_candidates: int, + seq_embeddings: Dict[str, SequenceEmbedding], + payload_features: Dict[str, torch.Tensor], + num_candidates: torch.Tensor, + total_uih_len: Optional[int] = None, + total_targets: Optional[int] = None, + ) -> torch.Tensor: + source_lengths = seq_embeddings[ + self._hstu_configs.uih_post_id_feature_name + ].lengths + source_timestamps = concat_2D_jagged( + max_seq_len=max_uih_len + max_candidates, + max_len_left=max_uih_len, + offsets_left=payload_features["uih_offsets"], + values_left=payload_features[ + self._hstu_configs.uih_action_time_feature_name + ].unsqueeze(-1), + max_len_right=max_candidates, + offsets_right=payload_features["candidate_offsets"], + values_right=payload_features[ + self._hstu_configs.candidates_querytime_feature_name + ].unsqueeze(-1), + kernel=self.hammer_kernel(), + ).squeeze(-1) + if total_targets is None: + total_targets = fx_total_targets(num_candidates) + if total_uih_len is None: + total_uih_len = source_timestamps.numel() - total_targets + embedding = seq_embeddings[ + self._hstu_configs.uih_post_id_feature_name + ].embedding + dtype = embedding.dtype + if (not self.is_inference) and self._bf16_training: + embedding = embedding.to(torch.bfloat16) + if torch.jit.is_scripting(): + # TorchScript does not support ``with torch.autocast(...)``. + # In script-mode inference the dense path is already in bf16 + # (move_sparse_output_to_device upcasts on the C++ side), so + # autocast is a no-op for the path the predictor exercises. + candidates_user_embeddings, _ = self._hstu_transducer( + max_uih_len=max_uih_len, + max_targets=max_candidates, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_embeddings=embedding, + seq_lengths=source_lengths, + seq_timestamps=source_timestamps, + seq_payloads=self._construct_payload( + payload_features=payload_features, + seq_embeddings=seq_embeddings, + ), + num_targets=num_candidates, + ) + else: + with torch.autocast( + "cuda", + dtype=torch.bfloat16, + enabled=(not self.is_inference) and self._bf16_training, + ): + candidates_user_embeddings, _ = self._hstu_transducer( + max_uih_len=max_uih_len, + max_targets=max_candidates, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_embeddings=embedding, + seq_lengths=source_lengths, + seq_timestamps=source_timestamps, + seq_payloads=self._construct_payload( + payload_features=payload_features, + seq_embeddings=seq_embeddings, + ), + num_targets=num_candidates, + ) + candidates_user_embeddings = candidates_user_embeddings.to(dtype) + + return candidates_user_embeddings + + def _item_forward( + self, + seq_embeddings: Dict[str, SequenceEmbedding], + ) -> torch.Tensor: # [L, D] + all_embeddings = torch.cat( + [ + seq_embeddings[name].embedding + for name in self._hstu_configs.item_embedding_feature_names + ], + dim=-1, + ) + item_embeddings = self._item_embedding_mlp(all_embeddings) + return item_embeddings + + def preprocess( + self, + uih_features: KeyedJaggedTensor, + candidates_features: KeyedJaggedTensor, + merged_sparse_features: Optional[KeyedJaggedTensor] = None, + ) -> Tuple[ + Dict[str, SequenceEmbedding], + Dict[str, torch.Tensor], + int, + torch.Tensor, + int, + torch.Tensor, + ]: + # Embedding lookup for uih + candidates. When the caller (the pipeline + # path) supplies the pre-merged KJT from the batch, feed it straight to + # the EmbeddingCollection: that keeps the lookup's input a plain getattr + # off the batch so TorchRec's TrainPipelineSparseDist can hoist its + # input_dist into the prefetch stage. Building it here (cat + + # from_lengths_sync's .sync()) is an "input modification" that makes + # TorchRec skip pipelining the embedding collection. + if merged_sparse_features is None: + merged_sparse_features = KeyedJaggedTensor.from_lengths_sync( + keys=uih_features.keys() + candidates_features.keys(), + values=torch.cat( + [uih_features.values(), candidates_features.values()], + dim=0, + ), + lengths=torch.cat( + [uih_features.lengths(), candidates_features.lengths()], + dim=0, + ), + ) + seq_embeddings_dict = self._embedding_collection(merged_sparse_features) + num_candidates = fx_mark_length_features( + candidates_features.lengths().view(len(candidates_features.keys()), -1) + )[0] + max_num_candidates = fx_infer_max_len(num_candidates) + uih_seq_lengths = uih_features[ + self._hstu_configs.uih_post_id_feature_name + ].lengths() + max_uih_len = fx_infer_max_len(uih_seq_lengths) + + # prepare payload features + payload_features: Dict[str, torch.Tensor] = {} + for ( + uih_feature_name, + candidate_feature_name, + ) in self._hstu_configs.merge_uih_candidate_feature_mapping: + if ( + candidate_feature_name + not in self._hstu_configs.item_embedding_feature_names + and uih_feature_name + not in self._hstu_configs.user_embedding_feature_names + ): + values_left = uih_features[uih_feature_name].values() + if self._is_inference and ( + candidate_feature_name + == self._hstu_configs.candidates_weight_feature_name + or candidate_feature_name + == self._hstu_configs.candidates_watchtime_feature_name + ): + total_candidates = torch.sum(num_candidates).item() + values_right = torch.zeros( + total_candidates, # pyre-ignore + dtype=torch.int64, + device=values_left.device, + ) + else: + values_right = candidates_features[candidate_feature_name].values() + payload_features[uih_feature_name] = values_left + payload_features[candidate_feature_name] = values_right + payload_features["uih_offsets"] = torch.ops.fbgemm.asynchronous_complete_cumsum( + uih_seq_lengths + ) + payload_features["candidate_offsets"] = ( + torch.ops.fbgemm.asynchronous_complete_cumsum(num_candidates) + ) + + seq_embeddings = { + k: SequenceEmbedding( + lengths=seq_embeddings_dict[k].lengths(), + embedding=seq_embeddings_dict[k].values(), + ) + for k in self._hstu_configs.user_embedding_feature_names + + self._hstu_configs.item_embedding_feature_names + } + + return ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) + + def main_forward( + self, + seq_embeddings: Dict[str, SequenceEmbedding], + payload_features: Dict[str, torch.Tensor], + max_uih_len: int, + uih_seq_lengths: torch.Tensor, + max_num_candidates: int, + num_candidates: torch.Tensor, + total_uih_len: Optional[int] = None, + total_targets: Optional[int] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + # Stash this batch's jagged FLOPs/sample for MetricsLogger to read. + # No D->H sync: the .item() happens once per metric_log_frequency in + # the trainer, not on every step. Eval-mode batches also produce a + # stash but the trainer only consumes it on train batches. + if not torch.jit.is_scripting(): + self._last_jagged_flops_per_sample = ( + self._compute_jagged_flops_per_sample( + uih_seq_lengths=uih_seq_lengths, + num_candidates=num_candidates, + ) + ) + + # merge uih and candidates embeddings + for ( + uih_feature_name, + candidate_feature_name, + ) in self._hstu_configs.merge_uih_candidate_feature_mapping: + if uih_feature_name in seq_embeddings: + seq_embeddings[uih_feature_name] = SequenceEmbedding( + lengths=uih_seq_lengths + num_candidates, + embedding=concat_2D_jagged( + max_seq_len=max_uih_len + max_num_candidates, + max_len_left=max_uih_len, + offsets_left=torch.ops.fbgemm.asynchronous_complete_cumsum( + uih_seq_lengths + ), + values_left=seq_embeddings[uih_feature_name].embedding, + max_len_right=max_num_candidates, + offsets_right=torch.ops.fbgemm.asynchronous_complete_cumsum( + num_candidates + ), + values_right=seq_embeddings[candidate_feature_name].embedding, + kernel=self.hammer_kernel(), + ), + ) + + with record_function("## item_forward ##"): + candidates_item_embeddings = self._item_forward( + seq_embeddings, + ) + with record_function("## user_forward ##"): + candidates_user_embeddings = self._user_forward( + max_uih_len=max_uih_len, + max_candidates=max_num_candidates, + seq_embeddings=seq_embeddings, + payload_features=payload_features, + num_candidates=num_candidates, + total_uih_len=total_uih_len, + total_targets=total_targets, + ) + with record_function("## multitask_module ##"): + supervision_labels, supervision_weights = ( + _get_supervision_labels_and_weights( + supervision_bitmasks=payload_features[ + self._hstu_configs.candidates_weight_feature_name + ], + watchtime_sequence=payload_features[ + self._hstu_configs.candidates_watchtime_feature_name + ], + task_configs=self._multitask_configs, + ) + ) + mt_target_preds, mt_target_labels, mt_target_weights, mt_losses = ( + self._multitask_module( + encoded_user_embeddings=candidates_user_embeddings, + item_embeddings=candidates_item_embeddings, + supervision_labels=supervision_labels, + supervision_weights=supervision_weights, + ) + ) + + aux_losses: Dict[str, torch.Tensor] = {} + if not self._is_inference and self.training: + for i, task in enumerate(self._multitask_configs): + aux_losses[task.task_name] = mt_losses[i] + + return ( + candidates_user_embeddings, + candidates_item_embeddings, + aux_losses, + mt_target_preds, + mt_target_labels, + mt_target_weights, + ) + + def forward( + self, + uih_features: KeyedJaggedTensor, + candidates_features: Optional[KeyedJaggedTensor] = None, + merged_sparse_features: Optional[KeyedJaggedTensor] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + # Pipeline mode: TorchRec fx-traces this forward (via DMP.module) and the + # pipeline calls it with the single `Samples` batch. Unpacking the KJTs + # here — rather than in the wrapper — makes the EmbeddingCollection's + # input `batch.merged_sparse_features` a getattr off the batch placeholder, + # which is what lets TrainPipelineSparseDist hoist the embedding input_dist + # into the prefetch stage. Guarded from TorchScript (inference path). + if not torch.jit.is_scripting() and self._pipeline_mode: + batch = uih_features + uih_features = batch.uih_features_kjt + candidates_features = batch.candidates_features_kjt + merged_sparse_features = batch.merged_sparse_features + + with record_function("## preprocess ##"): + ( + seq_embeddings, + payload_features, + max_uih_len, + uih_seq_lengths, + max_num_candidates, + num_candidates, + ) = self.preprocess( + uih_features=uih_features, + candidates_features=candidates_features, + merged_sparse_features=merged_sparse_features, + ) + + with record_function("## main_forward ##"): + return self.main_forward( + seq_embeddings=seq_embeddings, + payload_features=payload_features, + max_uih_len=max_uih_len, + uih_seq_lengths=uih_seq_lengths, + max_num_candidates=max_num_candidates, + num_candidates=num_candidates, + ) diff --git a/recommendation_v4/generative_recommenders/modules/dynamic_stu.py b/recommendation_v4/generative_recommenders/modules/dynamic_stu.py new file mode 100644 index 000000000..e1fe8ad16 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/dynamic_stu.py @@ -0,0 +1,304 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict +import abc +import contextlib +from typing import Any, Generator, Optional, Tuple + +import torch +from generative_recommenders.common import fx_infer_max_len +from generative_recommenders.modules.stu import STU +from generative_recommenders.ops.jagged_tensors import ( + hstu_concat_l2_embeddings, + hstu_split_l2_embeddings, +) + + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +@contextlib.contextmanager +# pyre-ignore[3] +def _freeze_rng_state() -> Generator[Any, None, None]: + rng_state = torch.get_rng_state() + if torch.cuda.is_available(): + cuda_rng_state = torch.cuda.get_rng_state() + try: + yield + finally: + if torch.cuda.is_available(): + # pyre-ignore[61] + torch.cuda.set_rng_state(cuda_rng_state) + torch.set_rng_state(rng_state) + + +class DynamicSTU(STU): + def __init__(self, stu: STU, is_inference: bool) -> None: + super().__init__(is_inference) + self._stu = stu + + @abc.abstractmethod + def _preprocess( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + int, + torch.Tensor, + int, + Optional[torch.Tensor], + ]: + pass + + @abc.abstractmethod + def _postprocess( + self, + stu_output: torch.Tensor, + ) -> torch.Tensor: + pass + + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + ( + x, + x_lengths, + x_offsets, + max_seq_len, + num_targets, + max_kv_caching_len, + kv_caching_lengths, + ) = self._preprocess( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + + stu_output = self._stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + + return self._postprocess( + stu_output=stu_output, + ) + + +class SDSTU(DynamicSTU): + def __init__( + self, + stu: STU, + is_inference: bool, + dropout_ratio: float = 0.5, + seed: int = 0, + ) -> None: + """ + Stochastic Depth STU + """ + super().__init__(stu=stu, is_inference=is_inference) + self._dropout_ratio: float = dropout_ratio + self._iter: int = 0 + self._seed: int = seed + self._skip_x: Optional[torch.Tensor] = None + + def _preprocess( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + int, + torch.Tensor, + int, + Optional[torch.Tensor], + ]: + if self.training: + with _freeze_rng_state(): + torch.manual_seed(self._iter + self._seed) + prob = torch.rand(1) + if prob.item() <= self._dropout_ratio: + new_x = torch.empty(size=(0, x.shape[1]), device=x.device) + self._skip_x = x + new_x_lengths = torch.zeros_like(x_lengths) + new_x_offsets = torch.zeros_like(x_offsets) + new_max_seq_len = 1 + else: + new_x = x + new_x_lengths = x_lengths + new_x_offsets = x_offsets + new_max_seq_len = max_seq_len + self._iter += 1 + else: + new_x = x + new_x_lengths = x_lengths + new_x_offsets = x_offsets + new_max_seq_len = max_seq_len + return ( + new_x, + new_x_lengths, + new_x_offsets, + new_max_seq_len, + num_targets, + max_kv_caching_len, + kv_caching_lengths, + ) + + def _postprocess( + self, + stu_output: torch.Tensor, + ) -> torch.Tensor: + if self.training and self._skip_x is not None: + ret = self._skip_x + self._skip_x = None + return ret + else: + return stu_output + + +@torch.fx.wrap +def _fx_unwrap_optional_tuple_tensor( + optional: Optional[Tuple[torch.Tensor, torch.Tensor, torch.Tensor]], +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert optional is not None, "Expected optional to be non-None" + return optional + + +class L2STU(DynamicSTU): + def __init__( + self, + stu: STU, + max_l2_len: int, + is_inference: bool, + contextual_seq_len: int = 0, + ) -> None: + """ + Stochastic Depth STU + """ + super().__init__(stu=stu, is_inference=is_inference) + self._max_l2_len: int = max_l2_len + self._contextual_seq_len: int = contextual_seq_len + self._saved_tensors: Optional[ + Tuple[torch.Tensor, torch.Tensor, torch.Tensor] + ] = None + self._runtime_max_l2_len: int = 0 + self._runtime_prefix_len: int = 0 + + def _preprocess( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + int, + torch.Tensor, + int, + Optional[torch.Tensor], + ]: + prefix_lengths = ( + x_lengths - self._max_l2_len - num_targets - self._contextual_seq_len + ) + prefix_lengths = torch.clamp(prefix_lengths, min=0) + prefix_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(prefix_lengths) + l2_lengths = x_lengths - prefix_lengths + l2_offsets = x_offsets - prefix_offsets + self._runtime_max_l2_len: int = fx_infer_max_len(l2_lengths) + self._runtime_prefix_len: int = fx_infer_max_len(prefix_lengths) + prefix_x, l2_x = hstu_split_l2_embeddings( + max_seq_len=max_seq_len, + x=x, + prefix_offsets=prefix_offsets, + l2_offsets=l2_offsets, + contextual_seq_len=self._contextual_seq_len, + kernel=self.hammer_kernel(), + ) + self._saved_tensors = ( + prefix_offsets, + prefix_x, + l2_offsets, + ) + return ( + l2_x, + l2_lengths, + l2_offsets, + self._runtime_max_l2_len, + num_targets, + max_kv_caching_len, + kv_caching_lengths, + ) + + def _postprocess( + self, + stu_output: torch.Tensor, + ) -> torch.Tensor: + ( + prefix_offsets, + prefix_x, + l2_offsets, + ) = _fx_unwrap_optional_tuple_tensor(self._saved_tensors) + self._saved_tensors = None + return hstu_concat_l2_embeddings( + max_prefix_len=self._runtime_prefix_len, + prefix_x=prefix_x, + prefix_offsets=prefix_offsets, + max_l2_len=self._runtime_max_l2_len, + l2_x=stu_output, + l2_offsets=l2_offsets, + contextual_seq_len=self._contextual_seq_len, + kernel=self.hammer_kernel(), + ) diff --git a/recommendation_v4/generative_recommenders/modules/hstu_transducer.py b/recommendation_v4/generative_recommenders/modules/hstu_transducer.py new file mode 100644 index 000000000..ce91a67c9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/hstu_transducer.py @@ -0,0 +1,330 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import logging +from typing import Dict, Optional, Tuple + +import torch +from generative_recommenders.common import fx_unwrap_optional_tensor, HammerModule +from generative_recommenders.modules.positional_encoder import HSTUPositionalEncoder +from generative_recommenders.modules.postprocessors import ( + L2NormPostprocessor, + OutputPostprocessor, +) +from generative_recommenders.modules.preprocessors import InputPreprocessor +from generative_recommenders.modules.stu import STU +from generative_recommenders.ops.jagged_tensors import split_2D_jagged +from torch.profiler import record_function + +logger: logging.Logger = logging.getLogger(__name__) +torch.fx.wrap("len") + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +@torch.fx.wrap +def default_seq_payload( + seq_payloads: Optional[Dict[str, torch.Tensor]], +) -> Dict[str, torch.Tensor]: + if seq_payloads is None: + return {} + else: + return torch.jit._unwrap_optional(seq_payloads) + + +class HSTUTransducer(HammerModule): + def __init__( + self, + stu_module: STU, + input_preprocessor: InputPreprocessor, + output_postprocessor: Optional[OutputPostprocessor] = None, + input_dropout_ratio: float = 0.0, + positional_encoder: Optional[HSTUPositionalEncoder] = None, + is_inference: bool = True, + return_full_embeddings: bool = False, + listwise: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._stu_module = stu_module + self._input_preprocessor: InputPreprocessor = input_preprocessor + self._output_postprocessor: OutputPostprocessor = ( + output_postprocessor + if output_postprocessor is not None + else L2NormPostprocessor(is_inference=is_inference) + ) + assert self._is_inference == self._input_preprocessor._is_inference, ( + f"input_preprocessor must have the same mode; self: {self._is_inference} vs input_preprocessor {self._input_preprocessor._is_inference}" + ) + self._positional_encoder: Optional[HSTUPositionalEncoder] = positional_encoder + self._input_dropout_ratio: float = input_dropout_ratio + self._return_full_embeddings: bool = return_full_embeddings + self._listwise_training: bool = listwise and self.is_train + + for name, m in self.named_modules(): + if "_stu_module" in name: + continue + elif isinstance(m, torch.nn.Linear): + torch.nn.init.xavier_normal_(m.weight) + elif isinstance(m, torch.nn.LayerNorm): + if m.weight.dim() >= 2: + torch.nn.init.xavier_normal_(m.weight) + if m.bias is not None and m.bias.dim() >= 2: + torch.nn.init.xavier_normal_(m.bias) + + def _preprocess( + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + ]: + seq_payloads = default_seq_payload(seq_payloads) + + with record_function("hstu_input_preprocessor"): + ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + output_seq_payloads, + ) = self._input_preprocessor( + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_timestamps=seq_timestamps, + seq_embeddings=seq_embeddings, + num_targets=num_targets, + seq_payloads=seq_payloads, + ) + + with record_function("hstu_positional_encoder"): + if self._positional_encoder is not None: + output_seq_embeddings = self._positional_encoder( + max_seq_len=output_max_seq_len, + seq_lengths=output_seq_lengths, + seq_offsets=output_seq_offsets, + seq_timestamps=output_seq_timestamps, + seq_embeddings=output_seq_embeddings, + num_targets=( + None if self._listwise_training else output_num_targets + ), + ) + + output_seq_embeddings = torch.nn.functional.dropout( + output_seq_embeddings, + p=self._input_dropout_ratio, + training=self.training, + ) + + return ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + output_seq_payloads, + ) + + def _hstu_compute( + self, + max_seq_len: int, + seq_lengths: torch.Tensor, + seq_offsets: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + ) -> torch.Tensor: + with record_function("hstu"): + seq_embeddings = self._stu_module( + max_seq_len=max_seq_len, + x=seq_embeddings, + x_lengths=seq_lengths, + x_offsets=seq_offsets, + num_targets=(None if self._listwise_training else num_targets), + ) + return seq_embeddings + + def _postprocess( + self, + max_seq_len: int, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[Optional[torch.Tensor], torch.Tensor]: + with record_function("hstu_output_postprocessor"): + if self._return_full_embeddings: + seq_embeddings = self._output_postprocessor( + seq_embeddings=seq_embeddings, + seq_timestamps=seq_timestamps, + seq_payloads=seq_payloads, + ) + uih_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + seq_lengths - num_targets + ) + candidates_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + num_targets + ) + _, candidate_embeddings = split_2D_jagged( + values=seq_embeddings, + max_seq_len=max_seq_len, + total_len_left=total_uih_len, + total_len_right=total_targets, + max_len_left=max_uih_len, + max_len_right=max_targets, + offsets_left=uih_offsets, + offsets_right=candidates_offsets, + kernel=self.hammer_kernel(), + ) + interleave_targets: bool = self._input_preprocessor.interleave_targets() + if interleave_targets: + candidate_embeddings = candidate_embeddings.view( + -1, 2, candidate_embeddings.size(-1) + )[:, 0, :] + if not self._return_full_embeddings: + _, candidate_timestamps = split_2D_jagged( + values=seq_timestamps.unsqueeze(-1), + max_seq_len=max_seq_len, + total_len_left=total_uih_len, + total_len_right=total_targets, + max_len_left=max_uih_len, + max_len_right=max_targets, + offsets_left=uih_offsets, + offsets_right=candidates_offsets, + kernel=self.hammer_kernel(), + ) + candidate_timestamps = candidate_timestamps.squeeze(-1) + if interleave_targets: + candidate_timestamps = candidate_timestamps.view(-1, 2)[:, 0] + candidate_embeddings = self._output_postprocessor( + seq_embeddings=candidate_embeddings, + seq_timestamps=candidate_timestamps, + seq_payloads=seq_payloads, + ) + + return ( + seq_embeddings if self._return_full_embeddings else None, + candidate_embeddings, + ) + + def forward( + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + ]: + orig_dtype = seq_embeddings.dtype + if not self._is_inference: + seq_embeddings = seq_embeddings.to(self._training_dtype) + + ( + max_seq_len, + total_uih_len, + total_targets, + seq_lengths, + seq_offsets, + seq_timestamps, + seq_embeddings, + num_targets, + seq_payloads, + ) = self._preprocess( + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_timestamps=seq_timestamps, + seq_embeddings=seq_embeddings, + num_targets=num_targets, + seq_payloads=seq_payloads, + ) + + encoded_embeddings = self._hstu_compute( + max_seq_len=max_seq_len, + seq_lengths=seq_lengths, + seq_offsets=seq_offsets, + seq_timestamps=seq_timestamps, + seq_embeddings=seq_embeddings, + num_targets=num_targets, + ) + + encoded_embeddings, encoded_candidate_embeddings = self._postprocess( + max_seq_len=max_seq_len, + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_embeddings=encoded_embeddings, + seq_timestamps=seq_timestamps, + num_targets=num_targets, + seq_payloads=seq_payloads, + ) + + if not self._is_inference: + encoded_candidate_embeddings = encoded_candidate_embeddings.to(orig_dtype) + if self._return_full_embeddings: + encoded_embeddings = fx_unwrap_optional_tensor(encoded_embeddings).to( + orig_dtype + ) + return ( + encoded_candidate_embeddings, + encoded_embeddings, + ) diff --git a/recommendation_v4/generative_recommenders/modules/multitask_module.py b/recommendation_v4/generative_recommenders/modules/multitask_module.py new file mode 100644 index 000000000..3cb11996f --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/multitask_module.py @@ -0,0 +1,288 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import abc +import logging +from dataclasses import dataclass +from enum import IntEnum +from typing import Callable, Dict, List, Optional, Tuple + +import numpy as np +import torch +import torch.nn.functional as F +from generative_recommenders.common import HammerModule + +logger: logging.Logger = logging.getLogger(__name__) + + +class MultitaskTaskType(IntEnum): + BINARY_CLASSIFICATION = 0 + REGRESSION = 1 + + +@dataclass +class TaskConfig: + task_name: str + task_weight: int + task_type: MultitaskTaskType + + +class MultitaskModule(HammerModule): + @abc.abstractmethod + def forward( + self, + encoded_user_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + supervision_labels: Dict[str, torch.Tensor], + supervision_weights: Dict[str, torch.Tensor], + ) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + """ + Computes multi-task predictions. + + Args: + encoded_user_embeddings: (L, D) x float. + item_embeddings: (L, D) x float. + supervision_labels: Dict[T, L] x float or int + supervision_weights: Dict[T', L] x float or int, T' <= T + Returns: + (T, L) x float, predictions, labels, weights, losses + """ + pass + + +def _compute_pred_and_logits( + prediction_module: torch.nn.Module, + encoded_user_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + task_offsets: List[int], + has_multiple_task_types: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + mt_logits = prediction_module(encoded_user_embeddings * item_embeddings).transpose( + 0, 1 + ) + mt_preds_list: List[torch.Tensor] = [] + for task_type in MultitaskTaskType: + logits = mt_logits[ + task_offsets[task_type] : task_offsets[task_type + 1], + :, + ] + if task_offsets[task_type + 1] - task_offsets[task_type] > 0: + if task_type == MultitaskTaskType.REGRESSION: + mt_preds_list.append(logits) + else: + mt_preds_list.append(F.sigmoid(logits)) + if has_multiple_task_types: + mt_preds: torch.Tensor = torch.concat(mt_preds_list, dim=0) + else: + mt_preds: torch.Tensor = mt_preds_list[0] + + return mt_preds, mt_logits + + +def _compute_labels_and_weights( + supervision_labels: Dict[str, torch.Tensor], + supervision_weights: Dict[str, torch.Tensor], + task_configs: List[TaskConfig], + device: torch.device, + dtype: torch.dtype = torch.float32, +) -> Tuple[torch.Tensor, torch.Tensor]: + first_label: torch.Tensor = list(supervision_labels.values())[0] + default_supervision_weight = torch.ones_like( + first_label, + dtype=dtype, + device=device, + ) + mt_lables_list: List[torch.Tensor] = [] + mt_weights_list: List[torch.Tensor] = [] + for task in task_configs: + mt_lables_list.append(supervision_labels[task.task_name]) + mt_weights_list.append( + supervision_weights.get(task.task_name, default_supervision_weight) + ) + if len(task_configs) > 1: + mt_labels = torch.stack(mt_lables_list, dim=0) + mt_weights = torch.stack(mt_weights_list, dim=0) + else: + mt_labels = mt_lables_list[0].unsqueeze(0) + mt_weights = mt_weights_list[0].unsqueeze(0) + return mt_labels, mt_weights + + +def _compute_loss( + task_offsets: List[int], + causal_multitask_weights: float, + mt_logits: torch.Tensor, + mt_labels: torch.Tensor, + mt_weights: torch.Tensor, + has_multiple_task_types: bool, +) -> torch.Tensor: + mt_losses_list: List[torch.Tensor] = [] + for task_type in MultitaskTaskType: + if task_offsets[task_type + 1] - task_offsets[task_type] > 0: + logits = mt_logits[ + task_offsets[task_type] : task_offsets[task_type + 1], + :, + ] + labels = mt_labels[ + task_offsets[task_type] : task_offsets[task_type + 1], + :, + ] + weights = mt_weights[ + task_offsets[task_type] : task_offsets[task_type + 1], + :, + ] + if task_type == MultitaskTaskType.REGRESSION: + mt_losses_list.append( + F.mse_loss(logits, labels, reduction="none") * weights + ) + else: + mt_losses_list.append( + F.binary_cross_entropy_with_logits( + input=logits, target=labels, reduction="none" + ) + * weights + ) + + if has_multiple_task_types: + mt_losses = torch.concat(mt_losses_list, dim=0) + else: + mt_losses = mt_losses_list[0] + mt_losses = ( + mt_losses.sum(-1) / mt_weights.sum(-1).clamp(min=1.0) * causal_multitask_weights + ) + return mt_losses + + +class DefaultMultitaskModule(MultitaskModule): + def __init__( + self, + task_configs: List[TaskConfig], + embedding_dim: int, + prediction_fn: Callable[[int, int], torch.nn.Module], + causal_multitask_weights: float, + is_inference: bool, + ) -> None: + super().__init__(is_inference) + assert sorted(task_configs, key=lambda x: x.task_type) == task_configs, ( + "task_configs must be sorted by task_type." + ) + assert len(task_configs) > 0, "task_configs must be non-empty." + self._task_configs: List[TaskConfig] = task_configs + self._task_offsets: List[int] = [0] * (len(MultitaskTaskType) + 1) + for task in self._task_configs: + self._task_offsets[task.task_type + 1] += 1 + self._has_multiple_task_types: bool = self._task_offsets.count(0) < len( + MultitaskTaskType + ) + self._task_offsets[1:] = np.cumsum(self._task_offsets[1:]).tolist() + self._causal_multitask_weights: float = causal_multitask_weights + self._prediction_module: torch.nn.Module = prediction_fn( + embedding_dim, len(task_configs) + ) + + def forward( + self, + encoded_user_embeddings: torch.Tensor, + item_embeddings: torch.Tensor, + supervision_labels: Dict[str, torch.Tensor], + supervision_weights: Dict[str, torch.Tensor], + ) -> Tuple[ + torch.Tensor, + Optional[torch.Tensor], + Optional[torch.Tensor], + Optional[torch.Tensor], + ]: + orig_dtype = encoded_user_embeddings.dtype + if not self._is_inference: + encoded_user_embeddings = encoded_user_embeddings.to(self._training_dtype) + item_embeddings = item_embeddings.to(self._training_dtype) + + if torch.jit.is_scripting(): + # Script-mode fast path: skip torch.autocast (unsupported in TS) + # and inline _compute_pred_and_logits to avoid its + # `torch.nn.Module` parameter annotation (TS only knows + # concrete module types). The dense module is already in bf16 + # at this point, so autocast is a no-op for the predictor path. + mt_logits = self._prediction_module( + encoded_user_embeddings * item_embeddings + ).transpose(0, 1) + mt_preds_list: List[torch.Tensor] = [] + # MultitaskTaskType is an IntEnum (BINARY_CLASSIFICATION=0, + # REGRESSION=1) but TorchScript treats it as an opaque Enum. + # Iterate by the integer task indices directly. + for task_type in range(len(self._task_offsets) - 1): + start = self._task_offsets[task_type] + end = self._task_offsets[task_type + 1] + logits = mt_logits[start:end, :] + if end - start > 0: + # 1 == MultitaskTaskType.REGRESSION + if task_type == 1: + mt_preds_list.append(logits) + else: + mt_preds_list.append(F.sigmoid(logits)) + if self._has_multiple_task_types: + mt_preds: torch.Tensor = torch.concat(mt_preds_list, dim=0) + else: + mt_preds: torch.Tensor = mt_preds_list[0] + return mt_preds, None, None, None + + with torch.autocast( + "cuda", + dtype=torch.bfloat16, + enabled=(not self.is_inference and self._training_dtype == torch.bfloat16), + ): + mt_preds, mt_logits = _compute_pred_and_logits( + prediction_module=self._prediction_module, + encoded_user_embeddings=encoded_user_embeddings, + item_embeddings=item_embeddings, + task_offsets=self._task_offsets, + has_multiple_task_types=self._has_multiple_task_types, + ) + + # losses are always computed in fp32 + mt_labels: Optional[torch.Tensor] = None + mt_weights: Optional[torch.Tensor] = None + mt_losses: Optional[torch.Tensor] = None + if not self._is_inference: + mt_labels, mt_weights = _compute_labels_and_weights( + supervision_labels=supervision_labels, + supervision_weights=supervision_weights, + task_configs=self._task_configs, + device=encoded_user_embeddings.device, + ) + mt_losses = _compute_loss( + task_offsets=self._task_offsets, + causal_multitask_weights=self._causal_multitask_weights, + mt_logits=mt_logits.to(mt_labels.dtype), + mt_labels=mt_labels, + mt_weights=mt_weights, + has_multiple_task_types=self._has_multiple_task_types, + ) + mt_preds = mt_preds.to(orig_dtype) + + return ( + mt_preds, + mt_labels, + mt_weights, + mt_losses, + ) diff --git a/recommendation_v4/generative_recommenders/modules/positional_encoder.py b/recommendation_v4/generative_recommenders/modules/positional_encoder.py new file mode 100644 index 000000000..99d904fd4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/positional_encoder.py @@ -0,0 +1,75 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from math import sqrt +from typing import Optional + +import torch +from generative_recommenders.common import HammerModule +from generative_recommenders.ops.position import add_timestamp_positional_embeddings + + +class HSTUPositionalEncoder(HammerModule): + def __init__( + self, + num_position_buckets: int, + num_time_buckets: int, + embedding_dim: int, + contextual_seq_len: int, + is_inference: bool = True, + ) -> None: + super().__init__(is_inference=is_inference) + self._embedding_dim: int = embedding_dim + self._contextual_seq_len: int = contextual_seq_len + self._position_embeddings_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.empty(num_position_buckets, embedding_dim).uniform_( + -sqrt(1.0 / num_position_buckets), + sqrt(1.0 / num_position_buckets), + ), + ) + self._timestamp_embeddings_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.empty(num_time_buckets + 1, embedding_dim).uniform_( + -sqrt(1.0 / num_time_buckets), + sqrt(1.0 / num_time_buckets), + ), + ) + + def forward( + self, + max_seq_len: int, + seq_lengths: torch.Tensor, + seq_offsets: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: Optional[torch.Tensor], + ) -> torch.Tensor: + seq_embeddings = add_timestamp_positional_embeddings( + alpha=self._embedding_dim**0.5, + max_seq_len=max_seq_len, + max_contextual_seq_len=self._contextual_seq_len, + position_embeddings_weight=self._position_embeddings_weight, + timestamp_embeddings_weight=self._timestamp_embeddings_weight, + seq_offsets=seq_offsets, + seq_lengths=seq_lengths, + seq_embeddings=seq_embeddings, + timestamps=seq_timestamps, + num_targets=num_targets, + interleave_targets=False, + kernel=self.hammer_kernel(), + ) + return seq_embeddings diff --git a/recommendation_v4/generative_recommenders/modules/postprocessors.py b/recommendation_v4/generative_recommenders/modules/postprocessors.py new file mode 100644 index 000000000..7958e3fa9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/postprocessors.py @@ -0,0 +1,182 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from abc import abstractmethod +from typing import Dict, List, Tuple + +import torch +from generative_recommenders.common import HammerModule, init_mlp_weights_optional_bias + + +@torch.fx.wrap +def _cast_dtype(t: torch.Tensor, dtype: torch.dtype) -> torch.Tensor: + if t.dtype != dtype: + return t.to(dtype) + return t + + +class OutputPostprocessor(HammerModule): + """An abstract class for post-processing user embeddings after HSTU layers.""" + + @abstractmethod + def forward( + self, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + """ + Args: + seq_embeddings: (L, D) + seq_timestamps: (L, ) + seq_payloads: str-keyed tensors. Implementation specific. + + Returns: + postprocessed seq_embeddings, (L, D) + """ + pass + + +class L2NormPostprocessor(OutputPostprocessor): + """Postprocesses user embeddings with l2 norm.""" + + def __init__(self, is_inference: bool = False) -> None: + super().__init__(is_inference=is_inference) + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + return seq_embeddings / torch.linalg.norm( + seq_embeddings, ord=2, dim=-1, keepdim=True + ).clamp(min=1e-6) + + +class LayerNormPostprocessor(OutputPostprocessor): + """Postprocesses user embeddings with layer norm.""" + + def __init__( + self, + embedding_dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + + self._layer_norm: torch.nn.Module = torch.nn.LayerNorm( + normalized_shape=[embedding_dim], eps=eps + ) + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + # pyre-fixme[6]: For 1st argument expected `dtype` but got `Union[dtype, + # Tensor, Module]`. + return self._layer_norm(seq_embeddings.to(self._layer_norm.weight.dtype)) + + +@torch.fx.wrap +def _unsqueeze_if_needed(t: torch.Tensor, embedding: torch.Tensor) -> torch.Tensor: + if embedding.dim() == 3: + return t.unsqueeze(0) + return t + + +class TimestampLayerNormPostprocessor(OutputPostprocessor): + """Postprocesses user embeddings with timestamp-based MLP -> layer norm.""" + + def __init__( + self, + embedding_dim: int, + time_duration_features: List[Tuple[int, int]], + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + + self._layer_norm: torch.nn.Module = torch.nn.LayerNorm( + normalized_shape=[embedding_dim], eps=eps + ) + self.register_buffer( + "_period_units", + torch.Tensor([f[0] for f in time_duration_features]).view(1, -1), + ) + self.register_buffer( + "_units_per_period", + torch.Tensor([f[1] for f in time_duration_features]).view(1, -1), + ) + self._time_feature_combiner: torch.nn.Module = torch.nn.Linear( + embedding_dim + 2 * len(time_duration_features), + embedding_dim, + ).apply(init_mlp_weights_optional_bias) + + def _concat_time_features( + self, + combined_embeddings: torch.Tensor, + timestamps: torch.Tensor, # [B] or [B, D] + ) -> torch.Tensor: + # concat time representation to combined embeddings + period_units = self._period_units + units_per_period = self._units_per_period + + timestamps = timestamps.unsqueeze(-1) + period_units = _unsqueeze_if_needed(period_units, combined_embeddings) + units_per_period = _unsqueeze_if_needed( + units_per_period, combined_embeddings + ).float() + # Compute time features in float32 to avoid bf16 precision loss through + # discontinuous floor/remainder ops, matching Inductor fusion behavior. + _units_elapsed_type: torch.dtype = combined_embeddings.dtype + _units_since_epoch = torch.div( + timestamps.float(), period_units.float(), rounding_mode="floor" + ) # [sum(N_i), num_time_features] or [B, N, num_time_features] + _units_elapsed = ( + (torch.remainder(_units_since_epoch, units_per_period) / units_per_period) + * 2 + * 3.14 + ) + _units_elapsed = torch.view_as_real( + torch.polar( + _cast_dtype(torch.ones_like(_units_elapsed), torch.float32), + _cast_dtype(_units_elapsed, torch.float32), + ) + ).flatten( + -2, -1 + ) # [sum(N_i), num_time_features * 2] or [B, N, num_time_features * 2] + _units_elapsed = _cast_dtype(_units_elapsed, _units_elapsed_type) + combined_embeddings = torch.cat([combined_embeddings, _units_elapsed], dim=-1) + return combined_embeddings + + def forward( + self, + seq_embeddings: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> torch.Tensor: + user_embeddings = self._time_feature_combiner( + self._concat_time_features(seq_embeddings, timestamps=seq_timestamps).to( + self._time_feature_combiner.weight.dtype # pyre-fixme[6]: For 1st argument expected `dtype` but got `Union[dtype, + # Tensor, Module]`. + ) + ) + return self._layer_norm(user_embeddings) diff --git a/recommendation_v4/generative_recommenders/modules/preprocessors.py b/recommendation_v4/generative_recommenders/modules/preprocessors.py new file mode 100644 index 000000000..dc7806bb4 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/preprocessors.py @@ -0,0 +1,334 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import abc +from math import sqrt +from typing import Dict, List, Optional, Tuple + +import torch +from generative_recommenders.common import ( + fx_unwrap_optional_tensor, + HammerModule, + init_mlp_weights_optional_bias, + jagged_to_padded_dense, +) +from generative_recommenders.modules.action_encoder import ActionEncoder +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged +from generative_recommenders.ops.layer_norm import LayerNorm, SwishLayerNorm + + +class InputPreprocessor(HammerModule): + """An abstract class for pre-processing sequence embeddings before HSTU layers.""" + + @abc.abstractmethod + def forward( + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + ]: + """ + Args: + max_uih_len: int + max_targets: int + total_uih_len: int + total_targets: int + seq_lengths: (B,) + seq_embeddings: (L, D) + seq_timestamps: (B, N) + num_targets: (B,) Optional. + seq_payloads: str-keyed tensors. Implementation specific. + + Returns: + (max_seq_len, total_uih_len, total_targets, lengths, offsets, timestamps, embeddings, num_targets, payloads) updated based on input preprocessor. + """ + pass + + def interleave_targets(self) -> bool: + return False + + +def get_contextual_input_embeddings( + seq_lengths: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + contextual_feature_to_max_length: Dict[str, int], + contextual_feature_to_min_uih_length: Dict[str, int], + dtype: torch.dtype, +) -> torch.Tensor: + padded_values: List[torch.Tensor] = [] + for key, max_len in contextual_feature_to_max_length.items(): + v = torch.flatten( + jagged_to_padded_dense( + values=seq_payloads[key].to(dtype), + offsets=[seq_payloads[key + "_offsets"]], + max_lengths=[max_len], + padding_value=0.0, + ), + 1, + 2, + ) + min_uih_length = contextual_feature_to_min_uih_length.get(key, 0) + if min_uih_length > 0: + v = v * (seq_lengths.view(-1, 1) >= min_uih_length) + padded_values.append(v) + return torch.cat(padded_values, dim=1) + + +class ContextualPreprocessor(InputPreprocessor): + def __init__( + self, + input_embedding_dim: int, + hidden_dim: int, + output_embedding_dim: int, + contextual_feature_to_max_length: Dict[str, int], + contextual_feature_to_min_uih_length: Dict[str, int], + action_embedding_dim: int = 8, + action_feature_name: str = "", + action_weights: Optional[List[int]] = None, + additional_embedding_features: List[str] = [], + action_embedding_init_std: float = 0.1, + is_inference: bool = True, + ) -> None: + super().__init__(is_inference=is_inference) + self._output_embedding_dim: int = output_embedding_dim + self._input_embedding_dim: int = input_embedding_dim + self._hidden_dim: int = hidden_dim + self._contextual_feature_to_max_length: Dict[str, int] = ( + contextual_feature_to_max_length + ) + self._max_contextual_seq_len: int = sum( + contextual_feature_to_max_length.values() + ) + self._contextual_feature_to_min_uih_length: Dict[str, int] = ( + contextual_feature_to_min_uih_length + ) + if self._max_contextual_seq_len > 0: + std = 1.0 * sqrt( + 2.0 / float(input_embedding_dim + self._output_embedding_dim) + ) + self._batched_contextual_linear_weights: torch.nn.Parameter = ( + torch.nn.Parameter( + torch.empty( + ( + self._max_contextual_seq_len, + input_embedding_dim, + self._output_embedding_dim, + ) + ).normal_(0.0, std) + ) + ) + self._batched_contextual_linear_bias: torch.nn.Parameter = ( + torch.nn.Parameter( + torch.empty( + (self._max_contextual_seq_len, self._output_embedding_dim) + ).fill_(0.0) + ) + ) + self._content_embedding_mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=self._input_embedding_dim, + out_features=self._hidden_dim, + ), + SwishLayerNorm(self._hidden_dim), + torch.nn.Linear( + in_features=self._hidden_dim, + out_features=self._output_embedding_dim, + ), + LayerNorm(self._output_embedding_dim), + ).apply(init_mlp_weights_optional_bias) + self._additional_embedding_features: List[str] = additional_embedding_features + self._additional_embedding_mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=self._input_embedding_dim + * len(additional_embedding_features), + out_features=self._hidden_dim, + ), + SwishLayerNorm(self._hidden_dim), + torch.nn.Linear( + in_features=self._hidden_dim, + out_features=self._output_embedding_dim, + ), + LayerNorm(self._output_embedding_dim), + ).apply(init_mlp_weights_optional_bias) + self._action_feature_name: str = action_feature_name + self._action_weights: Optional[List[int]] = action_weights + if self._action_weights is not None: + self._action_encoder: ActionEncoder = ActionEncoder( + action_feature_name=action_feature_name, + action_weights=self._action_weights, + action_embedding_dim=action_embedding_dim, + embedding_init_std=action_embedding_init_std, + is_inference=is_inference, + ) + self._action_embedding_mlp: torch.nn.Module = torch.nn.Sequential( + torch.nn.Linear( + in_features=self._action_encoder.output_embedding_dim, + out_features=self._hidden_dim, + ), + SwishLayerNorm(self._hidden_dim), + torch.nn.Linear( + in_features=self._hidden_dim, + out_features=self._output_embedding_dim, + ), + LayerNorm(self._output_embedding_dim), + ).apply(init_mlp_weights_optional_bias) + + def forward( # noqa C901 + self, + max_uih_len: int, + max_targets: int, + total_uih_len: int, + total_targets: int, + seq_lengths: torch.Tensor, + seq_timestamps: torch.Tensor, + seq_embeddings: torch.Tensor, + num_targets: torch.Tensor, + seq_payloads: Dict[str, torch.Tensor], + ) -> Tuple[ + int, + int, + int, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + Dict[str, torch.Tensor], + ]: + output_seq_embeddings = self._content_embedding_mlp(seq_embeddings) + if len(self._additional_embedding_features) > 0: + additional_embeddings = torch.cat( + [ + seq_payloads[feature] + for feature in self._additional_embedding_features + ], + dim=1, + ) + output_seq_embeddings = ( + output_seq_embeddings + + self._additional_embedding_mlp(additional_embeddings) + ) + max_seq_len = max_uih_len + max_targets + target_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(num_targets) + seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(seq_lengths) + uih_offsets = seq_offsets - target_offsets + if self._action_weights is not None: + action_embeddings = self._action_encoder( + max_uih_len=max_uih_len, + max_targets=max_targets, + uih_offsets=uih_offsets, + target_offsets=target_offsets, + seq_embeddings=seq_embeddings, + seq_payloads=seq_payloads, + ) + output_seq_embeddings = output_seq_embeddings + self._action_embedding_mlp( + action_embeddings + ) + + output_max_seq_len = max_seq_len + output_total_uih_len = total_uih_len + output_total_targets = total_targets + output_seq_lengths = seq_lengths + output_num_targets = num_targets + output_seq_timestamps = seq_timestamps + output_seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + output_seq_lengths + ) + # concat contextual embeddings + if self._max_contextual_seq_len > 0: + contextual_input_embeddings = get_contextual_input_embeddings( + seq_lengths=seq_lengths, + seq_payloads=seq_payloads, + contextual_feature_to_max_length=self._contextual_feature_to_max_length, + contextual_feature_to_min_uih_length=self._contextual_feature_to_min_uih_length, + dtype=seq_embeddings.dtype, + ) + contextual_embeddings = torch.baddbmm( + self._batched_contextual_linear_bias.view( + -1, 1, self._output_embedding_dim + ).to(contextual_input_embeddings.dtype), + contextual_input_embeddings.view( + -1, self._max_contextual_seq_len, self._input_embedding_dim + ).transpose(0, 1), + self._batched_contextual_linear_weights.to( + contextual_input_embeddings.dtype + ), + ).transpose(0, 1) + output_seq_embeddings = concat_2D_jagged( + max_seq_len=self._max_contextual_seq_len + output_max_seq_len, + values_left=fx_unwrap_optional_tensor(contextual_embeddings).reshape( + -1, self._output_embedding_dim + ), + values_right=output_seq_embeddings, + max_len_left=self._max_contextual_seq_len, + max_len_right=output_max_seq_len, + offsets_left=None, + offsets_right=output_seq_offsets, + kernel=self.hammer_kernel(), + ) + output_seq_timestamps = concat_2D_jagged( + max_seq_len=self._max_contextual_seq_len + output_max_seq_len, + values_left=torch.zeros( + (output_seq_lengths.size(0) * self._max_contextual_seq_len, 1), + dtype=output_seq_timestamps.dtype, + device=output_seq_timestamps.device, + ), + values_right=output_seq_timestamps.unsqueeze(-1), + max_len_left=self._max_contextual_seq_len, + max_len_right=output_max_seq_len, + offsets_left=None, + offsets_right=output_seq_offsets, + kernel=self.hammer_kernel(), + ).squeeze(-1) + output_max_seq_len = output_max_seq_len + self._max_contextual_seq_len + output_seq_lengths = output_seq_lengths + self._max_contextual_seq_len + output_seq_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + output_seq_lengths + ) + output_total_uih_len = ( + output_total_uih_len + + self._max_contextual_seq_len * output_seq_lengths.size(0) + ) + + return ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + seq_payloads, + ) diff --git a/recommendation_v4/generative_recommenders/modules/stu.py b/recommendation_v4/generative_recommenders/modules/stu.py new file mode 100644 index 000000000..45c6ea5f3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/stu.py @@ -0,0 +1,471 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict +import abc +from dataclasses import dataclass +from typing import List, Optional, Tuple + +import torch +from generative_recommenders.common import fx_unwrap_optional_tensor, HammerModule +from generative_recommenders.ops.hstu_attention import delta_hstu_mha +from generative_recommenders.ops.hstu_compute import ( + hstu_compute_output, + hstu_compute_uqvk, + hstu_preprocess_and_attention, +) +from generative_recommenders.ops.jagged_tensors import concat_2D_jagged, split_2D_jagged +from torch.autograd.profiler import record_function + + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +class STU(HammerModule, abc.ABC): + def cached_forward( + self, + delta_x: torch.Tensor, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + raise NotImplementedError + + @abc.abstractmethod + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + pass + + +@dataclass +class STULayerConfig: + embedding_dim: int + num_heads: int + hidden_dim: int + attention_dim: int + output_dropout_ratio: float = 0.3 + causal: bool = True + target_aware: bool = True + max_attn_len: Optional[int] = None + attn_alpha: Optional[float] = None + use_group_norm: bool = False + recompute_normed_x: bool = True + recompute_uvqk: bool = True + recompute_y: bool = True + sort_by_length: bool = True + contextual_seq_len: int = 0 + + +@torch.fx.wrap +def _update_kv_cache( + max_seq_len: int, + seq_offsets: torch.Tensor, + k: Optional[torch.Tensor], + v: Optional[torch.Tensor], + max_kv_caching_len: int, + kv_caching_lengths: Optional[torch.Tensor], + orig_k_cache: Optional[torch.Tensor], + orig_v_cache: Optional[torch.Tensor], + orig_max_kv_caching_len: int, + orig_kv_caching_offsets: Optional[torch.Tensor], +) -> Tuple[Optional[torch.Tensor], Optional[torch.Tensor], int, Optional[torch.Tensor]]: + if kv_caching_lengths is not None: + kv_caching_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + kv_caching_lengths + ) + delta_offsets = seq_offsets - kv_caching_offsets + k_cache, _ = split_2D_jagged( + max_seq_len=max_seq_len, + values=fx_unwrap_optional_tensor(k).flatten(1, 2), + max_len_left=None, + max_len_right=None, + offsets_left=kv_caching_offsets, + offsets_right=delta_offsets, + ) + v_cache, _ = split_2D_jagged( + max_seq_len=max_seq_len, + values=fx_unwrap_optional_tensor(v).flatten(1, 2), + max_len_left=None, + max_len_right=None, + offsets_left=kv_caching_offsets, + offsets_right=delta_offsets, + ) + if max_kv_caching_len == 0: + max_kv_caching_len = int(kv_caching_lengths.max().item()) + return ( + k_cache, + v_cache, + max_kv_caching_len, + kv_caching_offsets, + ) + else: + return ( + orig_k_cache, + orig_v_cache, + orig_max_kv_caching_len, + orig_kv_caching_offsets, + ) + + +@torch.fx.wrap +def _construct_full_kv( + delta_k: torch.Tensor, + delta_v: torch.Tensor, + k_cache: torch.Tensor, + v_cache: torch.Tensor, + max_kv_caching_len: int, + kv_caching_offsets: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, int, torch.Tensor]: + L, _ = delta_k.shape + B = kv_caching_offsets.shape[0] - 1 + delta_size = L // B + full_k = concat_2D_jagged( + max_seq_len=max_kv_caching_len + delta_size, + values_left=k_cache, + values_right=delta_k, + max_len_left=max_kv_caching_len, + max_len_right=delta_size, + offsets_left=kv_caching_offsets, + offsets_right=None, + ) + full_v = concat_2D_jagged( + max_seq_len=max_kv_caching_len + delta_size, + values_left=v_cache, + values_right=delta_v, + max_len_left=max_kv_caching_len, + max_len_right=delta_size, + offsets_left=kv_caching_offsets, + offsets_right=None, + ) + full_kv_caching_offsets = kv_caching_offsets + delta_size * torch.arange( + B + 1, device=delta_k.device + ) + return ( + full_k, + full_v, + max_kv_caching_len + delta_size, + full_kv_caching_offsets, + ) + + +class STULayer(STU): + max_kv_caching_len: int + k_cache: Optional[torch.Tensor] + v_cache: Optional[torch.Tensor] + kv_caching_offsets: Optional[torch.Tensor] + + def __init__( + self, + config: STULayerConfig, + is_inference: bool = False, + ) -> None: + super().__init__( + is_inference=is_inference, + ) + self.reset_kv_cache() + self._num_heads: int = config.num_heads + self._embedding_dim: int = config.embedding_dim + self._hidden_dim: int = config.hidden_dim + self._attention_dim: int = config.attention_dim + self._output_dropout_ratio: float = config.output_dropout_ratio + self._target_aware: bool = config.target_aware + self._causal: bool = config.causal + self._max_attn_len: int = config.max_attn_len or 0 + self._attn_alpha: float = config.attn_alpha or 1.0 / (self._attention_dim**0.5) + self._use_group_norm: bool = config.use_group_norm + self._recompute_normed_x: bool = config.recompute_normed_x + self._recompute_uvqk: bool = config.recompute_uvqk + self._recompute_y: bool = config.recompute_y + self._sort_by_length: bool = config.sort_by_length + self._contextual_seq_len: int = config.contextual_seq_len + + self._uvqk_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.empty( + ( + self._embedding_dim, + (self._hidden_dim * 2 + self._attention_dim * 2) * self._num_heads, + ) + ), + ) + torch.nn.init.xavier_uniform_(self._uvqk_weight) + self._uvqk_beta: torch.nn.Parameter = torch.nn.Parameter( + torch.zeros( + (self._hidden_dim * 2 + self._attention_dim * 2) * self._num_heads, + ), + ) + self._input_norm_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.ones((self._embedding_dim,)), + ) + self._input_norm_bias: torch.nn.Parameter = torch.nn.Parameter( + torch.zeros((self._embedding_dim,)), + ) + self._output_weight = torch.nn.Parameter( + torch.empty( + ( + self._hidden_dim * self._num_heads * 3, + self._embedding_dim, + ) + ), + ) + torch.nn.init.xavier_uniform_(self._output_weight) + output_norm_shape: int = ( + self._hidden_dim * self._num_heads + if not self._use_group_norm + else self._num_heads + ) + self._output_norm_weight: torch.nn.Parameter = torch.nn.Parameter( + torch.ones((output_norm_shape,)), + ) + self._output_norm_bias: torch.nn.Parameter = torch.nn.Parameter( + torch.zeros((output_norm_shape,)), + ) + + def reset_kv_cache(self) -> None: + self.k_cache = None + self.v_cache = None + self.kv_caching_offsets = None + self.max_kv_caching_len = 0 + + def update_kv_cache( + self, + max_seq_len: int, + seq_offsets: torch.Tensor, + k: Optional[torch.Tensor], + v: Optional[torch.Tensor], + max_kv_caching_len: int, + kv_caching_lengths: Optional[torch.Tensor], + ) -> None: + self.k_cache, self.v_cache, self.max_kv_caching_len, self.kv_caching_offsets = ( + _update_kv_cache( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + k=k, + v=v, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + orig_k_cache=self.k_cache, + orig_v_cache=self.v_cache, + orig_max_kv_caching_len=self.max_kv_caching_len, + orig_kv_caching_offsets=self.kv_caching_offsets, + ) + ) + + def construct_full_kv( + self, + delta_k: torch.Tensor, + delta_v: torch.Tensor, + ) -> Tuple[torch.Tensor, torch.Tensor, int, torch.Tensor]: + return _construct_full_kv( + delta_k=delta_k, + delta_v=delta_v, + k_cache=fx_unwrap_optional_tensor(self.k_cache), + v_cache=fx_unwrap_optional_tensor(self.v_cache), + max_kv_caching_len=self.max_kv_caching_len, + kv_caching_offsets=self.kv_caching_offsets, + ) + + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + with record_function("## stu_preprocess_and_attention ##"): + u, attn_output, k, v = hstu_preprocess_and_attention( + x=x, + norm_weight=self._input_norm_weight.to(x.dtype), + norm_bias=self._input_norm_bias.to(x.dtype), + norm_eps=1e-6, + num_heads=self._num_heads, + attn_dim=self._attention_dim, + hidden_dim=self._hidden_dim, + uvqk_weight=self._uvqk_weight.to(x.dtype), + uvqk_bias=self._uvqk_beta.to(x.dtype), + max_seq_len=max_seq_len, + seq_offsets=x_offsets, + attn_alpha=self._attn_alpha, + causal=self._causal, + num_targets=num_targets if self._target_aware else None, + max_attn_len=self._max_attn_len, + contextual_seq_len=self._contextual_seq_len, + recompute_uvqk_in_backward=self._recompute_uvqk, + recompute_normed_x_in_backward=self._recompute_normed_x, + sort_by_length=self._sort_by_length, + prefill=kv_caching_lengths is not None, + kernel=self.hammer_kernel(), + ) + + self.update_kv_cache( + max_seq_len=max_seq_len, + seq_offsets=x_offsets, + k=k, + v=v, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + + with record_function("## stu_compute_output ##"): + return hstu_compute_output( + attn=attn_output, + u=u, + x=x, + norm_weight=self._output_norm_weight.to(x.dtype), + norm_bias=self._output_norm_bias.to(x.dtype), + norm_eps=1e-6, + dropout_ratio=self._output_dropout_ratio, + output_weight=self._output_weight.to(x.dtype), + group_norm=self._use_group_norm, + num_heads=self._num_heads, + linear_dim=self._hidden_dim, + concat_u=True, + concat_x=True, + mul_u_activation_type="none", + training=self.training, + kernel=self.hammer_kernel(), + recompute_y_in_backward=self._recompute_y, + ) + + def cached_forward( + self, + delta_x: torch.Tensor, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + with record_function("## stu_compute_uqvk ##"): + delta_u, delta_q, delta_k, delta_v = hstu_compute_uqvk( + x=delta_x, + norm_weight=self._input_norm_weight.to(delta_x.dtype), + norm_bias=self._input_norm_bias.to(delta_x.dtype), + norm_eps=1e-6, + num_heads=self._num_heads, + attn_dim=self._attention_dim, + hidden_dim=self._hidden_dim, + uvqk_weight=self._uvqk_weight.to(delta_x.dtype), + uvqk_bias=self._uvqk_beta.to(delta_x.dtype), + kernel=self.hammer_kernel(), + ) + k, v, max_seq_len, seq_offsets = self.construct_full_kv( + delta_k=delta_k.flatten(1, 2), + delta_v=delta_v.flatten(1, 2), + ) + self.update_kv_cache( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + k=k, + v=v, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + k = k.view(-1, self._num_heads, self._attention_dim) + v = v.view(-1, self._num_heads, self._hidden_dim) + with record_function("## delta_hstu_mha ##"): + delta_attn_output = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=self._attn_alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if self._target_aware else None, + max_attn_len=self._max_attn_len, + contextual_seq_len=self._contextual_seq_len, + kernel=self.hammer_kernel(), + ).view(-1, self._hidden_dim * self._num_heads) + with record_function("## stu_compute_output ##"): + return hstu_compute_output( + attn=delta_attn_output, + u=delta_u, + x=delta_x, + norm_weight=self._output_norm_weight.to(delta_x.dtype), + norm_bias=self._output_norm_bias.to(delta_x.dtype), + norm_eps=1e-6, + dropout_ratio=self._output_dropout_ratio, + output_weight=self._output_weight.to(delta_x.dtype), + group_norm=self._use_group_norm, + num_heads=self._num_heads, + linear_dim=self._hidden_dim, + concat_u=True, + concat_x=True, + mul_u_activation_type="none", + training=self.training, + kernel=self.hammer_kernel(), + recompute_y_in_backward=self._recompute_y, + ) + + +class STUStack(STU): + def __init__( + self, + stu_list: List[STU], + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._stu_layers: torch.nn.ModuleList = torch.nn.ModuleList(modules=stu_list) + + def forward( + self, + x: torch.Tensor, + x_lengths: torch.Tensor, + x_offsets: torch.Tensor, + max_seq_len: int, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + for layer in self._stu_layers: + x = layer( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + return x + + def cached_forward( + self, + delta_x: torch.Tensor, + num_targets: torch.Tensor, + max_kv_caching_len: int = 0, + kv_caching_lengths: Optional[torch.Tensor] = None, + ) -> torch.Tensor: + for layer in self._stu_layers: + delta_x = layer.cached_forward( # pyre-ignore [29] + delta_x=delta_x, + num_targets=num_targets, + max_kv_caching_len=max_kv_caching_len, + kv_caching_lengths=kv_caching_lengths, + ) + return delta_x diff --git a/recommendation_v4/generative_recommenders/modules/tests/action_encoder_test.py b/recommendation_v4/generative_recommenders/modules/tests/action_encoder_test.py new file mode 100644 index 000000000..184b314ea --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/action_encoder_test.py @@ -0,0 +1,113 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.modules.action_encoder import ActionEncoder + + +class ActionEncoderTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_forward(self) -> None: + device = torch.device("cuda") + action_embedding_dim = 32 + action_weights = [1, 2, 4, 8, 16] + watchtime_to_action_thresholds_and_weights = [ + (30, 32), + (60, 64), + (100, 128), + ] + num_action_types = len(action_weights) + len( + watchtime_to_action_thresholds_and_weights + ) + combined_action_weights = action_weights + [ + x[1] for x in watchtime_to_action_thresholds_and_weights + ] + enabled_actions = [ + [0], + [0, 1], + [1, 3, 4], + [1, 2, 3, 4], + [1, 2], + [2], + ] + watchtimes = [40, 20, 110, 31, 26, 55] + for i, wt in enumerate(watchtimes): + for j, w in enumerate(watchtime_to_action_thresholds_and_weights): + if wt > w[0]: + enabled_actions[i].append(j + len(action_weights)) + actions = [ + sum([combined_action_weights[t] for t in x]) for x in enabled_actions + ] + + encoder = ActionEncoder( + watchtime_feature_name="watchtimes", + action_feature_name="actions", + action_weights=action_weights, + watchtime_to_action_thresholds_and_weights=watchtime_to_action_thresholds_and_weights, + action_embedding_dim=action_embedding_dim, + is_inference=False, + ).to(device) + + seq_lengths = [6, 3] + seq_offsets = [0, 6, 9] + num_targets = [2, 1] + uih_offsets = [0, 4, 6] + target_offsets = [0, 2, 3] + seq_embeddings = torch.rand(9, 128, device=device) + action_embeddings = encoder( + max_uih_len=4, + max_targets=2, + uih_offsets=torch.tensor(uih_offsets, device=device), + target_offsets=torch.tensor(target_offsets, device=device), + seq_embeddings=seq_embeddings, + seq_payloads={ + "watchtimes": torch.tensor(watchtimes, device=device), + "actions": torch.tensor(actions, device=device), + }, + ) + self.assertEqual( + action_embeddings.shape, (9, action_embedding_dim * num_action_types) + ) + for b in range(len(seq_lengths)): + b_start = seq_offsets[b] + b_end = seq_offsets[b + 1] + u_start = uih_offsets[b] + for j in range(b_start, b_end): + embedding = action_embeddings[j].view(num_action_types, -1) + for atype in range(num_action_types): + if b_end - j <= num_targets[b]: + torch.testing.assert_allclose( + embedding[atype], + encoder._target_action_embedding_table.view( + num_action_types, -1 + )[atype], + ) + else: + if atype in enabled_actions[j - b_start + u_start]: + torch.testing.assert_allclose( + embedding[atype], + encoder._action_embedding_table[atype], + ) + else: + torch.testing.assert_allclose( + embedding[atype], torch.zeros_like(embedding[atype]) + ) + action_embeddings.sum().backward() diff --git a/recommendation_v4/generative_recommenders/modules/tests/content_encoder_test.py b/recommendation_v4/generative_recommenders/modules/tests/content_encoder_test.py new file mode 100644 index 000000000..e67656388 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/content_encoder_test.py @@ -0,0 +1,74 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.modules.content_encoder import ContentEncoder + + +class ContentEncoderTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + def test_forward(self) -> None: + device = torch.device("cuda") + input_embedding_dim = 32 + additional_embedding_dim = 64 + enrich_embedding_dim = 16 + encoder = ContentEncoder( + input_embedding_dim=input_embedding_dim, + additional_content_features={ + "a0": additional_embedding_dim, + "a1": additional_embedding_dim, + }, + target_enrich_features={ + "t0": enrich_embedding_dim, + "t1": enrich_embedding_dim, + }, + is_inference=False, + ).to(device) + seq_lengths = [6, 3] + num_targets = [2, 1] + uih_offsets = [0, 4, 6] + target_offsets = [0, 2, 3] + seq_embeddings = torch.rand( + sum(seq_lengths), input_embedding_dim, device=device + ).requires_grad_(True) + seq_payloads = { + "a0": torch.rand( + sum(seq_lengths), additional_embedding_dim, device=device + ).requires_grad_(True), + "a1": torch.rand( + sum(seq_lengths), additional_embedding_dim, device=device + ).requires_grad_(True), + "t0": torch.rand( + sum(num_targets), enrich_embedding_dim, device=device + ).requires_grad_(True), + "t1": torch.rand( + sum(num_targets), enrich_embedding_dim, device=device + ).requires_grad_(True), + } + content_embeddings = encoder( + max_uih_len=4, + max_targets=2, + uih_offsets=torch.tensor(uih_offsets, device=device), + target_offsets=torch.tensor(target_offsets, device=device), + seq_embeddings=seq_embeddings, + seq_payloads=seq_payloads, + ) + content_embeddings.sum().backward() diff --git a/recommendation_v4/generative_recommenders/modules/tests/contextual_interleave_preprocessor_test.py b/recommendation_v4/generative_recommenders/modules/tests/contextual_interleave_preprocessor_test.py new file mode 100644 index 000000000..c3202072c --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/contextual_interleave_preprocessor_test.py @@ -0,0 +1,499 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable +from generative_recommenders.modules.action_encoder import ActionEncoder +from generative_recommenders.modules.content_encoder import ContentEncoder +from generative_recommenders.modules.contextual_interleave_preprocessor import ( + ContextualInterleavePreprocessor, +) +from generative_recommenders.modules.contextualize_mlps import ( + ParameterizedContextualizedMLP, + SimpleContextualizedMLP, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class ContextualInterleavePreprocessorTest(unittest.TestCase): + # pyre-ignore + @given( + enable_interleaving=st.sampled_from([True, False]), + enable_pmlp=st.sampled_from([True, False]), + is_train=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=50, deadline=None) + def test_forward( + self, + enable_interleaving: bool, + enable_pmlp: bool, + is_train: bool, + dtype: torch.dtype, + ) -> None: + device = torch.device("cuda") + + input_embedding_dim = 64 + output_embedding_dim = 32 + action_embedding_dim = 16 + action_encoder_hidden_dim = 256 + content_encoder_hidden_dim = 128 + contextual_len = 3 + + content_encoder = ContentEncoder( + input_embedding_dim=input_embedding_dim, + additional_content_features={ + "a0": input_embedding_dim, + "a1": input_embedding_dim, + }, + target_enrich_features={ + "t0": input_embedding_dim, + "t1": input_embedding_dim, + }, + is_inference=False, + ).to(device) + action_embedding_dim = 32 + action_weights = [1, 2, 4, 8, 16] + watchtime_to_action_thresholds_and_weights = [ + (30, 32), + (60, 64), + (100, 128), + ] + action_encoder = ActionEncoder( + watchtime_feature_name="watchtimes", + action_feature_name="actions", + action_weights=action_weights, + watchtime_to_action_thresholds_and_weights=watchtime_to_action_thresholds_and_weights, + action_embedding_dim=action_embedding_dim, + is_inference=False, + ).to(device) + + preprocessor = ContextualInterleavePreprocessor( + input_embedding_dim=input_embedding_dim, + output_embedding_dim=output_embedding_dim, + contextual_feature_to_max_length={"c_0": 1, "c_1": 2}, + contextual_feature_to_min_uih_length={"c_1": 4}, + pmlp_contextual_dropout_ratio=0.2, + content_encoder=content_encoder, + content_contextualize_mlp_fn=lambda in_dim, + out_dim, + contextual_dim, + is_inference: ParameterizedContextualizedMLP( + contextual_embedding_dim=contextual_dim, + sequential_input_dim=in_dim, + sequential_output_dim=out_dim, + hidden_dim=content_encoder_hidden_dim, + is_inference=is_inference, + ) + if enable_pmlp + else SimpleContextualizedMLP( + sequential_input_dim=in_dim, + sequential_output_dim=out_dim, + hidden_dim=content_encoder_hidden_dim, + is_inference=is_inference, + ), + action_encoder=action_encoder, + action_contextualize_mlp_fn=lambda in_dim, + out_dim, + contextual_dim, + is_inference: ParameterizedContextualizedMLP( + contextual_embedding_dim=contextual_dim, + sequential_input_dim=in_dim, + sequential_output_dim=out_dim, + hidden_dim=action_encoder_hidden_dim, + is_inference=is_inference, + ) + if enable_pmlp + else SimpleContextualizedMLP( + sequential_input_dim=in_dim, + sequential_output_dim=out_dim, + hidden_dim=action_encoder_hidden_dim, + is_inference=is_inference, + ), + enable_interleaving=enable_interleaving, + is_inference=False, + ).to(device) + preprocessor.set_training_dtype(dtype) + if not is_train: + preprocessor.eval() + + # inputs + seq_lengths = [6, 3] + num_targets = [2, 1] + seq_embeddings = torch.rand( + (sum(seq_lengths), input_embedding_dim), + device=device, + dtype=dtype, + ) + seq_timestamps = torch.tensor( + [1, 2, 3, 4, 5, 6, 10, 20, 30], + device=device, + ) + watchtimes = [40, 20, 110, 31, 26, 55] + actions = [1, 3, 26, 30, 6, 4] + ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + _, + ) = preprocessor( + max_uih_len=4, + max_targets=2, + total_uih_len=sum(seq_lengths) - sum(num_targets), + total_targets=sum(num_targets), + seq_lengths=torch.tensor(seq_lengths, device=device), + seq_timestamps=seq_timestamps, + seq_embeddings=seq_embeddings, + seq_payloads={ + # contextual + "c_0": torch.rand((2, input_embedding_dim), device=device, dtype=dtype), + "c_0_offsets": torch.tensor([0, 1, 1], device=device), + "c_1": torch.rand((4, input_embedding_dim), device=device, dtype=dtype), + "c_1_offsets": torch.tensor([0, 2, 3], device=device), + # action + "watchtimes": torch.tensor(watchtimes, device=device), + "actions": torch.tensor(actions, device=device), + # content + "a0": torch.rand_like(seq_embeddings).requires_grad_(True), + "a1": torch.rand_like(seq_embeddings).requires_grad_(True), + "t0": torch.rand( + sum(num_targets), input_embedding_dim, device=device, dtype=dtype + ).requires_grad_(True), + "t1": torch.rand( + sum(num_targets), input_embedding_dim, device=device, dtype=dtype + ).requires_grad_(True), + }, + num_targets=torch.tensor(num_targets, device=device), + ) + if enable_interleaving: + if is_train: + expected_output_seq_lengths = [ + 2 * s + contextual_len for s in seq_lengths + ] + expected_max_seq_len = max(expected_output_seq_lengths) + expected_output_num_targets = [2 * s for s in num_targets] + expected_seq_embedding_size = ( + sum(expected_output_seq_lengths), + output_embedding_dim, + ) + expected_seq_timestamps_size = (sum(expected_output_seq_lengths),) + expected_output_seq_timestamps = [ + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 5, + 6, + 6, + 0, + 0, + 0, + 10, + 10, + 20, + 20, + 30, + 30, + ] + else: + expected_output_seq_lengths = [ + 2 * s - n + contextual_len for s, n in zip(seq_lengths, num_targets) + ] + expected_max_seq_len = max(expected_output_seq_lengths) + expected_output_num_targets = num_targets + expected_seq_embedding_size = ( + sum(expected_output_seq_lengths), + output_embedding_dim, + ) + expected_seq_timestamps_size = (sum(expected_output_seq_lengths),) + expected_output_seq_timestamps = [ + 0, + 0, + 0, + 1, + 1, + 2, + 2, + 3, + 3, + 4, + 4, + 5, + 6, + 0, + 0, + 0, + 10, + 10, + 20, + 20, + 30, + ] + else: + expected_output_seq_lengths = [s + contextual_len for s in seq_lengths] + expected_max_seq_len = max(expected_output_seq_lengths) + expected_output_num_targets = num_targets + expected_seq_embedding_size = ( + sum(expected_output_seq_lengths), + output_embedding_dim, + ) + expected_seq_timestamps_size = (sum(expected_output_seq_lengths),) + expected_output_seq_timestamps = [ + 0, + 0, + 0, + 1, + 2, + 3, + 4, + 5, + 6, + 0, + 0, + 0, + 10, + 20, + 30, + ] + + self.assertEqual(output_max_seq_len, expected_max_seq_len) + self.assertEqual(output_seq_lengths.tolist(), expected_output_seq_lengths) + torch.testing.assert_allclose( + torch.ops.fbgemm.asynchronous_complete_cumsum(output_seq_lengths), + output_seq_offsets, + ) + self.assertEqual(output_num_targets.tolist(), expected_output_num_targets) + self.assertEqual( + output_seq_embeddings.size(), + expected_seq_embedding_size, + ) + self.assertEqual( + output_seq_timestamps.size(), + expected_seq_timestamps_size, + ) + self.assertEqual( + output_seq_timestamps.tolist(), + expected_output_seq_timestamps, + ) + + # test combine embeddings + batch_size = 10 + max_uih_len = 100 + max_targets = 20 + max_seq_len = max_uih_len + max_targets + seq_lengths = torch.randint(0, max_uih_len, (batch_size,), device=device) + total_uih_len = int(seq_lengths.sum().item()) + num_targets = torch.randint(1, max_targets, (batch_size,), device=device) + total_targets = int(num_targets.sum().item()) + seq_lengths = seq_lengths + num_targets + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(seq_lengths, dim=0) + total_seq_len = int(torch.sum(seq_lengths).item()) + seq_timestamps = torch.randint(0, 1000000, (total_seq_len,), device=device) + content_embeddings = torch.rand( + (total_seq_len, output_embedding_dim), + device=device, + ).requires_grad_(True) + action_embeddings = torch.rand( + (total_seq_len, output_embedding_dim), + device=device, + ).requires_grad_(True) + contextual_embeddings = torch.rand( + (total_seq_len, 3 * output_embedding_dim), device=device + ).requires_grad_(True) + ( + output_max_seq_len, + output_total_uih_len, + output_total_targets, + output_seq_lengths, + output_seq_offsets, + output_seq_timestamps, + output_seq_embeddings, + output_num_targets, + ) = preprocessor.combine_embeddings( + max_uih_len=max_uih_len, + max_targets=max_targets, + total_uih_len=total_uih_len, + total_targets=total_targets, + seq_lengths=seq_lengths, + seq_timestamps=seq_timestamps, + content_embeddings=content_embeddings, + action_embeddings=action_embeddings, + contextual_embeddings=contextual_embeddings, + num_targets=num_targets, + ) + seq_embeddings = action_embeddings + content_embeddings + if enable_interleaving: + if is_train: + self.assertEqual( + output_max_seq_len, + max_seq_len * 2 + contextual_len, + ) + self.assertEqual( + output_total_uih_len, + total_uih_len * 2 + contextual_len * batch_size, + ) + self.assertEqual( + output_total_targets, + total_targets * 2, + ) + torch.testing.assert_allclose( + output_seq_lengths, seq_lengths * 2 + contextual_len + ) + torch.testing.assert_allclose(output_num_targets, num_targets * 2) + else: + self.assertEqual( + output_max_seq_len, + max_uih_len * 2 + max_targets + contextual_len, + ) + self.assertEqual( + output_total_uih_len, + total_uih_len * 2 + contextual_len * batch_size, + ) + self.assertEqual( + output_total_targets, + total_targets, + ) + torch.testing.assert_allclose( + output_seq_lengths, seq_lengths * 2 - num_targets + contextual_len + ) + torch.testing.assert_allclose(output_num_targets, num_targets) + else: + self.assertEqual( + output_max_seq_len, + max_seq_len + contextual_len, + ) + self.assertEqual( + output_total_uih_len, + total_uih_len + contextual_len * batch_size, + ) + self.assertEqual( + output_total_targets, + total_targets, + ) + torch.testing.assert_allclose( + output_seq_lengths, seq_lengths + contextual_len + ) + torch.testing.assert_allclose(output_num_targets, num_targets) + for b in range(batch_size): + input_start = int(seq_offsets[b].item()) + input_end = int(seq_offsets[b + 1].item()) + output_start = int(output_seq_offsets[b].item()) + output_end = int(output_seq_offsets[b + 1].item()) + input_targets = int(num_targets[b].item()) + output_targets = int(output_num_targets[b].item()) + torch.testing.assert_allclose( + output_seq_timestamps[output_start : output_start + contextual_len], + torch.zeros(3, device=device), + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_start : output_start + contextual_len + ].view(-1), + contextual_embeddings[b], + ) + if enable_interleaving: + torch.testing.assert_allclose( + output_seq_timestamps[ + output_start + contextual_len : output_end - output_targets + ].view(-1, 2)[:, 0], + seq_timestamps[input_start : input_end - input_targets], + ) + torch.testing.assert_allclose( + output_seq_timestamps[ + output_start + contextual_len : output_end - output_targets + ].view(-1, 2)[:, 1], + seq_timestamps[input_start : input_end - input_targets], + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_start + contextual_len : output_end - output_targets + ].view(-1, 2, output_embedding_dim)[:, 0, :], + content_embeddings[input_start : input_end - input_targets], + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_start + contextual_len : output_end - output_targets + ].view(-1, 2, output_embedding_dim)[:, 1, :], + action_embeddings[input_start : input_end - input_targets], + ) + if is_train: + torch.testing.assert_allclose( + output_seq_timestamps[ + output_end - output_targets : output_end + ].view(-1, 2)[:, 0], + seq_timestamps[input_end - input_targets : input_end], + ) + torch.testing.assert_allclose( + output_seq_timestamps[ + output_end - output_targets : output_end + ].view(-1, 2)[:, 1], + seq_timestamps[input_end - input_targets : input_end], + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_end - output_targets : output_end + ].view(-1, 2, output_embedding_dim)[:, 0, :], + content_embeddings[input_end - input_targets : input_end], + ) + torch.testing.assert_allclose( + output_seq_embeddings[ + output_end - output_targets : output_end + ].view(-1, 2, output_embedding_dim)[:, 1, :], + action_embeddings[input_end - input_targets : input_end], + ) + else: + torch.testing.assert_allclose( + output_seq_timestamps[output_end - output_targets : output_end], + seq_timestamps[input_end - input_targets : input_end], + ) + torch.testing.assert_allclose( + output_seq_embeddings[output_end - output_targets : output_end], + content_embeddings[input_end - input_targets : input_end], + ) + else: + torch.testing.assert_allclose( + output_seq_timestamps[output_start + contextual_len : output_end], + seq_timestamps[input_start:input_end], + ) + torch.testing.assert_allclose( + output_seq_embeddings[output_start + contextual_len : output_end], + seq_embeddings[input_start:input_end], + ) diff --git a/recommendation_v4/generative_recommenders/modules/tests/dynamic_stu_test.py b/recommendation_v4/generative_recommenders/modules/tests/dynamic_stu_test.py new file mode 100644 index 000000000..c1c598f1f --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/dynamic_stu_test.py @@ -0,0 +1,279 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import copy +import unittest +from typing import List + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel, set_dev_mode +from generative_recommenders.modules.dynamic_stu import L2STU, SDSTU +from generative_recommenders.modules.stu import STU, STULayer, STULayerConfig, STUStack +from hypothesis import given, settings, strategies as st, Verbosity + + +class DynamicStuTest(unittest.TestCase): + # pyre-ignore + @given( + causal=st.sampled_from([True]), + num_layers=st.sampled_from([2]), + num_heads=st.sampled_from([2]), + max_uih_len=st.sampled_from([300]), + batch_size=st.sampled_from([8]), + embedding_dim=st.sampled_from([16]), + attention_dim=st.sampled_from([32]), + linear_hidden_dim=st.sampled_from([64]), + has_multiple_targets=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None) + def test_l2_stu( + self, + causal: bool, + num_layers: int, + num_heads: int, + max_uih_len: int, + batch_size: int, + embedding_dim: int, + attention_dim: int, + linear_hidden_dim: int, + has_multiple_targets: bool, + contextual_seq_len: int, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + l3_stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + l3_stu: List[STU] = [ + L2STU( + stu=STUStack( + stu_list=l3_stu_layers, + is_inference=False, + ), + max_l2_len=100, + contextual_seq_len=contextual_seq_len, + is_inference=False, + ) + ] + l2_stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + l3_stu + l2_stu: List[STU] = [ + L2STU( + stu=STUStack( + stu_list=l2_stu_layers, + is_inference=False, + ), + max_l2_len=200, + contextual_seq_len=contextual_seq_len, + is_inference=False, + ) + ] + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + l2_stu + stu = STUStack( + stu_list=stu_layers, + is_inference=False, + ).to(device) + stu.recursive_setattr("_hammer_kernel", HammerKernel.TRITON) + + x_lengths = torch.randint(max_uih_len + 1, (batch_size,), device=device) + x_lengths = x_lengths + contextual_seq_len + max_seq_len = max_uih_len + contextual_seq_len + max_targets = 20 + num_targets = torch.randint(1, max_targets, size=(batch_size,), device=device) + if has_multiple_targets: + x_lengths = x_lengths + num_targets + max_seq_len = max_seq_len + max_targets + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu().item()) + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + dtype=dtype, + ).requires_grad_(True) + stu_output = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + dout = torch.randn_like(stu_output) + stu_output.backward(dout) + self.assertTrue(stu_output.shape == x.shape) + + # pyre-ignore + @given( + causal=st.sampled_from([True]), + num_layers=st.sampled_from([2]), + num_heads=st.sampled_from([2]), + max_uih_len=st.sampled_from([300]), + batch_size=st.sampled_from([8]), + embedding_dim=st.sampled_from([16]), + attention_dim=st.sampled_from([32]), + linear_hidden_dim=st.sampled_from([64]), + has_multiple_targets=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + dropout_ratio=st.sampled_from([0.0, 0.3, 1.0]), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None) + def test_sd_stu( + self, + causal: bool, + num_layers: int, + num_heads: int, + max_uih_len: int, + batch_size: int, + embedding_dim: int, + attention_dim: int, + linear_hidden_dim: int, + has_multiple_targets: bool, + contextual_seq_len: int, + dropout_ratio: float, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + stu = STUStack( + stu_list=stu_layers, + is_inference=False, + ).to(device) + sd_stu = SDSTU( + stu=copy.deepcopy(stu), + dropout_ratio=dropout_ratio, + is_inference=False, + ).to(device) + stu.recursive_setattr("_hammer_kernel", HammerKernel.PYTORCH) + sd_stu.recursive_setattr("_hammer_kernel", HammerKernel.PYTORCH) + x_lengths = torch.randint(max_uih_len + 1, (batch_size,), device=device) + x_lengths = x_lengths + contextual_seq_len + max_seq_len = max_uih_len + contextual_seq_len + max_targets = 20 + num_targets = torch.randint(1, max_targets, size=(batch_size,), device=device) + if has_multiple_targets: + x_lengths = x_lengths + num_targets + max_seq_len = max_seq_len + max_targets + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu().item()) + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + dtype=dtype, + ).requires_grad_(True) + stu_output = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + dout = torch.randn_like(stu_output) + stu_output.backward(dout) + assert x.grad is not None + d_x, x.grad = x.grad.detach().clone(), None + x = x.detach().clone().requires_grad_(True) + sd_stu_output = sd_stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + dout = dout.detach().clone() + sd_stu_output.backward(dout) + d_sd_x = x.grad.detach().clone() + + self.assertTrue(sd_stu_output.shape == x.shape) + if dropout_ratio == 0.0: + torch.testing.assert_close(stu_output, sd_stu_output) + torch.testing.assert_close(d_x, d_sd_x) + if dropout_ratio == 1.0: + torch.testing.assert_close(x, sd_stu_output) diff --git a/recommendation_v4/generative_recommenders/modules/tests/multitask_module_test.py b/recommendation_v4/generative_recommenders/modules/tests/multitask_module_test.py new file mode 100644 index 000000000..66f2db185 --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/multitask_module_test.py @@ -0,0 +1,233 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest +from typing import Dict, List, Tuple + +import torch +from generative_recommenders.common import gpu_unavailable, set_dev_mode +from generative_recommenders.modules.multitask_module import ( + DefaultMultitaskModule, + MultitaskTaskType, + TaskConfig, +) +from generative_recommenders.ops.layer_norm import SwishLayerNorm +from hypothesis import given, settings, strategies as st, Verbosity + + +_task_configs: List[List[TaskConfig]] = [ + [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + ], + [ + TaskConfig( + task_name="vvp", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], + [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_like", + task_weight=2, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_follow", + task_weight=4, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + ], + [ + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.REGRESSION, + ), + TaskConfig( + task_name="vvp", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], + [ + TaskConfig( + task_name="type_1", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], + [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_like", + task_weight=2, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_follow", + task_weight=4, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.REGRESSION, + ), + TaskConfig( + task_name="vvp", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], + [ + TaskConfig( + task_name="is_click", + task_weight=1, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_like", + task_weight=2, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="is_follow", + task_weight=4, + task_type=MultitaskTaskType.BINARY_CLASSIFICATION, + ), + TaskConfig( + task_name="rating", + task_weight=1, + task_type=MultitaskTaskType.REGRESSION, + ), + TaskConfig( + task_name="vvp", + task_weight=2, + task_type=MultitaskTaskType.REGRESSION, + ), + ], +] + + +def _get_random_supervision_labels_and_weights( + num_examples: int, + task_configs: List[TaskConfig], + device: torch.device, +) -> Tuple[Dict[str, torch.Tensor], Dict[str, torch.Tensor]]: + supervision_labels: Dict[str, torch.Tensor] = {} + supervision_weights: Dict[str, torch.Tensor] = {} + for task in task_configs: + if task.task_type == MultitaskTaskType.REGRESSION: + supervision_labels[task.task_name] = torch.randn( + num_examples, device=device + ).to(torch.float32) + else: + supervision_labels[task.task_name] = torch.randint( + 0, + 11, + (num_examples,), + device=device, + ).to(torch.float32) + + return supervision_labels, supervision_weights + + +class MultiTaskModuleTest(unittest.TestCase): + # pyre-ignore + @given( + task_config_idx=st.sampled_from(range(len(_task_configs))), + training=st.booleans(), + is_inference=st.booleans(), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=50, deadline=None) + def test_default_multitask_module( + self, + task_config_idx: int, + training: bool, + is_inference: bool, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + + L = 200 + embedding_dim = 64 + causal_multitask_weights = 0.3 + + task_configs: List[TaskConfig] = _task_configs[task_config_idx] + task_configs.sort(key=lambda x: x.task_type) + multitask_module = DefaultMultitaskModule( + task_configs=task_configs, + embedding_dim=embedding_dim, + prediction_fn=lambda in_dim, num_tasks: torch.nn.Sequential( + torch.nn.Linear(in_features=in_dim, out_features=512), + SwishLayerNorm(512), + torch.nn.Linear(in_features=512, out_features=num_tasks), + ), + causal_multitask_weights=causal_multitask_weights, + is_inference=is_inference, + ).to(device) + multitask_module.set_training_dtype(dtype) + supervision_labels, supervision_weights = ( + _get_random_supervision_labels_and_weights( + num_examples=L, + task_configs=task_configs, + device=device, + ) + ) + encoded_user_embeddings = torch.rand(L, embedding_dim, device=device) + item_embeddings = torch.rand(L, embedding_dim, device=device) + + ( + mt_preds, + mt_labels, + mt_weights, + mt_losses, + ) = multitask_module( + encoded_user_embeddings=encoded_user_embeddings, + item_embeddings=item_embeddings, + supervision_labels=supervision_labels, + supervision_weights=supervision_weights, + ) + + self.assertEqual(mt_preds.size(), (len(task_configs), L)) + if not is_inference: + self.assertEqual(mt_labels.size(), (len(task_configs), L)) + self.assertEqual(mt_weights.size(), (len(task_configs), L)) + if training: + self.assertEqual(mt_losses.size(), (len(task_configs),)) diff --git a/recommendation_v4/generative_recommenders/modules/tests/stu_test.py b/recommendation_v4/generative_recommenders/modules/tests/stu_test.py new file mode 100644 index 000000000..f6440e55e --- /dev/null +++ b/recommendation_v4/generative_recommenders/modules/tests/stu_test.py @@ -0,0 +1,453 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import copy +import unittest +from typing import List + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel, set_dev_mode +from generative_recommenders.modules.stu import STU, STULayer, STULayerConfig, STUStack +from generative_recommenders.ops.jagged_tensors import split_2D_jagged +from hypothesis import given, settings, strategies as st, Verbosity + + +def _inplace_swap( + batch_size: int, + x: torch.Tensor, + swap_from: torch.Tensor, + swap_to: torch.Tensor, +) -> torch.Tensor: + for i in range(batch_size): + tmp = x[i, swap_from[i], :].detach().clone() + x[i, swap_from[i], :] = x[i, swap_to[i], :] + x[i, swap_to[i], :] = tmp + return x + + +class StuTest(unittest.TestCase): + # pyre-ignore + @given( + causal=st.sampled_from([True]), + num_layers=st.sampled_from([2]), + num_heads=st.sampled_from([1, 2]), + max_uih_len=st.sampled_from([20, 64]), + batch_size=st.sampled_from([8]), + embedding_dim=st.sampled_from([16]), + attention_dim=st.sampled_from([32]), + linear_hidden_dim=st.sampled_from([64]), + has_multiple_targets=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + use_group_norm=st.sampled_from([True, False]), + recompute_uvqk_in_backward=st.sampled_from([True, False]), + recompute_normed_x_in_backward=st.sampled_from([True, False]), + recompute_y_in_backward=st.sampled_from([True, False]), + empty_inputs=st.sampled_from([False]), + dtype=st.sampled_from( + [torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=100, deadline=None) + def test_triton( + self, + causal: bool, + num_layers: int, + num_heads: int, + max_uih_len: int, + batch_size: int, + embedding_dim: int, + attention_dim: int, + linear_hidden_dim: int, + has_multiple_targets: bool, + contextual_seq_len: int, + use_group_norm: bool, + recompute_uvqk_in_backward: bool, + recompute_normed_x_in_backward: bool, + recompute_y_in_backward: bool, + empty_inputs: bool, # test the case where all the seqlen in the batch are 0 + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=has_multiple_targets, + max_attn_len=None, + attn_alpha=None, + use_group_norm=use_group_norm, + recompute_normed_x=recompute_normed_x_in_backward, + recompute_uvqk=recompute_uvqk_in_backward, + recompute_y=recompute_y_in_backward, + sort_by_length=True, + contextual_seq_len=contextual_seq_len, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + stu = STUStack( + stu_list=stu_layers, + is_inference=False, + ).to(device) + stu.recursive_setattr("_hammer_kernel", HammerKernel.PYTORCH) + stu_triton = copy.deepcopy(stu) + stu_triton.recursive_setattr("_hammer_kernel", HammerKernel.TRITON) + + if empty_inputs: + x_lengths = torch.zeros(batch_size, dtype=torch.int32, device=device) + num_targets = torch.zeros(batch_size, dtype=torch.int32, device=device) + contextual_seq_len = 0 + max_seq_len = 16 + else: + x_lengths = torch.randint(max_uih_len + 1, (batch_size,), device=device) + x_lengths = x_lengths + contextual_seq_len + max_seq_len = max_uih_len + contextual_seq_len + max_targets = 20 + num_targets = torch.randint( + 1, max_targets, size=(batch_size,), device=device + ) + if has_multiple_targets: + x_lengths = x_lengths + num_targets + max_seq_len = max_seq_len + max_targets + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu().item()) + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + dtype=dtype, + ).requires_grad_(True) + x_triton = x.clone().detach().requires_grad_() + stu_output = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + stu_triton_output = stu_triton( + x=x_triton, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + atol = 5e-3 if dtype == torch.bfloat16 else None + rtol = 1e-2 if dtype == torch.bfloat16 else None + torch.testing.assert_close(stu_triton_output, stu_output, atol=atol, rtol=rtol) + dout = torch.randn_like(stu_output) + stu_output.backward(dout) + dout = dout.detach().clone() + stu_triton_output.backward(dout) + torch.testing.assert_close(x.grad, x_triton.grad, atol=atol, rtol=rtol) + + # pyre-ignore + @given( + dtype=st.sampled_from( + [torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @unittest.skipIf(*gpu_unavailable) + @settings(verbosity=Verbosity.verbose, max_examples=8, deadline=None) + def test_target_invariance( + self, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + num_layers = 2 + num_heads = 2 + max_seq_len = 32 + batch_size = 8 + embedding_dim = 16 + attention_dim = 32 + linear_hidden_dim = 32 + causal = True + use_group_norm = False + recompute_normed_x_in_backward = False + recompute_uvqk_in_backward = False + recompute_y_in_backward = False + max_attn_len = None + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=causal, + target_aware=True, + max_attn_len=max_attn_len, + attn_alpha=None, + use_group_norm=use_group_norm, + recompute_normed_x=recompute_normed_x_in_backward, + recompute_uvqk=recompute_uvqk_in_backward, + recompute_y=recompute_y_in_backward, + sort_by_length=True, + contextual_seq_len=0, + ), + is_inference=False, + ) + for _ in range(num_layers) + ] + stu = STUStack( + stu_list=stu_layers, + is_inference=False, + ).to(device) + + x_lengths = torch.randint( + low=2, high=max_seq_len + 1, size=(batch_size,), device=device + ) + num_targets = torch.randint(low=2, high=10, size=(batch_size,), device=device) + x_lengths = x_lengths + num_targets + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu()) + + swap_from = torch.remainder( + torch.randint(20, (batch_size,), device=device), num_targets + ) + swap_to = torch.remainder( + torch.randint(20, (batch_size,), device=device), num_targets + ) + swap_from = x_lengths - 1 - swap_from + swap_to = x_lengths - 1 - swap_to + max_seq_len = int(x_lengths.max().item()) + + # forward() + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + dtype=dtype, + ).requires_grad_(True) + stu_output = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + stu_output_dense = torch.ops.fbgemm.jagged_to_padded_dense( + values=stu_output, + offsets=[x_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + + # swapped forward(). + dense_x = torch.ops.fbgemm.jagged_to_padded_dense( + x.detach(), + [x_offsets], + [max_seq_len], + ) + swapped_dense_x = _inplace_swap(batch_size, dense_x, swap_from, swap_to) + swapped_x = torch.ops.fbgemm.dense_to_jagged( + swapped_dense_x, + [x_offsets], + )[0].requires_grad_(True) + swapped_stu_output = stu( + x=swapped_x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + swapped_stu_output_dense = torch.ops.fbgemm.jagged_to_padded_dense( + values=swapped_stu_output, + offsets=[x_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + + # backward + dout = torch.randn_like(stu_output_dense) + stu_output_dense.backward(dout) + dout = dout.detach().clone() + swapped_stu_output_dense.backward( + _inplace_swap(batch_size, dout, swap_from, swap_to) + ) + + swapped_swapped_stu_output_dense = _inplace_swap( + batch_size, swapped_stu_output_dense, swap_from, swap_to + ) + torch.testing.assert_close(stu_output_dense, swapped_swapped_stu_output_dense) + + # backward + torch.testing.assert_close( + torch.ops.fbgemm.jagged_to_padded_dense( + swapped_x.grad, + [x_offsets], + [max_seq_len], + ), + _inplace_swap( + batch_size, + torch.ops.fbgemm.jagged_to_padded_dense( + x.grad, + [x_offsets], + [max_seq_len], + ), + swap_from, + swap_to, + ), + ) + + # pyre-ignore[56] + @given( + num_layers=st.sampled_from([1, 2, 4]), + num_heads=st.sampled_from([1, 4]), + max_uih_len=st.sampled_from([20, 128]), + batch_size=st.sampled_from([4, 8]), + embedding_dim=st.sampled_from([32]), + attention_dim=st.sampled_from([16]), + linear_hidden_dim=st.sampled_from([64]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings(verbosity=Verbosity.verbose, max_examples=20, deadline=None) + @unittest.skipIf(*gpu_unavailable) + @torch.inference_mode() + def test_cached_forward( + self, + num_layers: int, + num_heads: int, + max_uih_len: int, + batch_size: int, + embedding_dim: int, + attention_dim: int, + linear_hidden_dim: int, + contextual_seq_len: int, + ) -> None: + set_dev_mode(True) + device = torch.device("cuda") + + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + + use_group_norm = False + recompute_normed_x_in_backward = False + recompute_uvqk_in_backward = False + recompute_y_in_backward = False + max_attn_len = None + stu_layers: List[STU] = [ + STULayer( + config=STULayerConfig( + embedding_dim=embedding_dim, + num_heads=num_heads, + hidden_dim=linear_hidden_dim, + attention_dim=attention_dim, + output_dropout_ratio=0.0, + causal=True, + target_aware=True, + max_attn_len=max_attn_len, + attn_alpha=None, + use_group_norm=use_group_norm, + recompute_normed_x=recompute_normed_x_in_backward, + recompute_uvqk=recompute_uvqk_in_backward, + recompute_y=recompute_y_in_backward, + sort_by_length=True, + contextual_seq_len=contextual_seq_len, + ), + is_inference=True, + ) + for _ in range(num_layers) + ] + stu = STUStack( + stu_list=stu_layers, + is_inference=True, + ).to(device) + stu.recursive_setattr("_hammer_kernel", HammerKernel.TRITON) + stu.eval() + + x_lengths = torch.randint( + max_uih_len, max_uih_len + 1, (batch_size,), device=device + ) + x_lengths = x_lengths + contextual_seq_len + max_seq_len = max_uih_len + contextual_seq_len + delta_size = 20 + max_targets = delta_size * 2 + num_targets = torch.randint( + delta_size, max_targets + 1, size=(batch_size,), device=device + ) + x_lengths = x_lengths + num_targets + contextual_seq_len + max_seq_len = max_seq_len + max_targets + contextual_seq_len + x_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(x_lengths) + total_seq_len = int(x_offsets[-1].cpu().item()) + x = torch.randn( + int(total_seq_len), + embedding_dim, + device=device, + ).requires_grad_(True) + + # default forward(). + ref_y = stu( + x=x, + x_lengths=x_lengths, + x_offsets=x_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets, + ) + prime_lengths = x_lengths - delta_size + prime_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(prime_lengths) + _, ref_delta_y = split_2D_jagged( + max_seq_len=max_seq_len, + values=ref_y, + max_len_left=None, + max_len_right=delta_size, + offsets_left=prime_offsets, + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + + # cached forward(). + prime_x, delta_x = split_2D_jagged( + max_seq_len=max_seq_len, + values=x, + max_len_left=None, + max_len_right=delta_size, + offsets_left=prime_offsets, + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + _ = stu( + x=prime_x, + x_lengths=prime_lengths, + x_offsets=prime_offsets, + max_seq_len=max_seq_len, + num_targets=num_targets - delta_size, + max_kv_caching_len=max_seq_len - delta_size, + kv_caching_lengths=x_lengths - delta_size, + ) + delta_y = stu.cached_forward( + delta_x=delta_x, + num_targets=num_targets, + ) + + torch.testing.assert_close(ref_delta_y, delta_y) diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/addmm_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/addmm_bench.py new file mode 100644 index 000000000..b1be3a803 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/addmm_bench.py @@ -0,0 +1,174 @@ +# pyre-unsafe +import time +from typing import List, Optional, Tuple + +import click +import pandas as pd +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_addmm import ( + triton_addmm_fwd, + triton_addmm_fwd_tma_persistent, + triton_addmm_fwd_tma_ws_persistent_tlx, + triton_addmm_fwd_tma_ws_tlx, +) +from generative_recommenders.ops.utils import is_sm100 + +try: + # @manual=//triton:triton + import triton.language.extra.tlx as tlx # type: ignore + + HAS_TLX = True +except ImportError: + tlx = None + HAS_TLX = False + + +def get_kernel(provider: str) -> HammerKernel: + if provider == "triton": + return HammerKernel.TRITON + elif provider == "pytorch": + return HammerKernel.PYTORCH + else: + raise ValueError(f"Unknown provider {provider}") + + +def get_dtype(dtype: str) -> torch.dtype: + if dtype == "bfloat16": + return torch.bfloat16 + elif dtype == "float32": + return torch.float32 + elif dtype == "float16": + return torch.float16 + else: + raise ValueError(f"Not supported dtype {dtype}") + + +@click.command() +@click.option("--m", type=int, default=0) +@click.option("--k", type=int, default=4096) +@click.option("--n", type=int, default=4096) +@click.option("--dtype", type=str, default="bfloat16") +@click.option("--return-result", type=bool, default=False) +@click.option("--broadcast-y", type=bool, is_flag=True, default=False) +def main( + m: int, + k: int, + n: int, + dtype: str, + return_result: bool, + broadcast_y: bool, +) -> Optional[Tuple[List[triton.testing.Benchmark], List[pd.DataFrame]]]: + if m == 0: + batch_sizes = [64, 128, 256, 512] + else: + batch_sizes = [m] + line_vals = [ + "pytorch", + "triton", + "triton_tma_persistent", + "triton_tma_persistent_ws", + ] + line_names = [ + "PyTorch", + "Triton", + "Triton TMA Persistent", + "Triton TMA Persistent WS", + ] + styles = [ + ("red", "-"), + ("green", "-"), + ("orange", "-"), + ("purple", "-"), + ] + if is_sm100() and HAS_TLX: # tmem is only supported on Blackwell + line_vals.append("triton_tma_ws_tlx") + line_names.append("Triton TMA WS TLX") + styles.append(("cyan", "-")) + line_vals.append("triton_tma_persistent_ws_tlx") + line_names.append("Triton TMA Persistent WS TLX") + styles.append(("magenta", "-")) + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["batch_size"], + x_vals=batch_sizes, + line_arg="provider", + line_vals=line_vals, + line_names=line_names, + styles=styles, + ylabel="ms", + plot_name=f"addmm-K-{k}-N-{n}-mode-{mode}-dtype-{dtype}-broadcast_y-{broadcast_y}", + args={ + "K": k, + "N": n, + "dtype": dtype, + "broadcast_y": broadcast_y, + }, + ) + for mode in ["fwd"] + ] + + @triton.testing.perf_report(configs) + def bench_addmm( + batch_size: int, + K: int, + N: int, + dtype: str, + provider: str, + broadcast_y: bool, + ) -> float: + warmup = 20 + rep = 2000 + x = torch.randn( + (batch_size, K), dtype=get_dtype(dtype), device=torch.device("cuda") + ).requires_grad_(True) + weight = torch.randn( + (N, K), dtype=get_dtype(dtype), device=torch.device("cuda") + ).requires_grad_(True) + if broadcast_y: + y = torch.randn( + (N), dtype=get_dtype(dtype), device=torch.device("cuda") + ).requires_grad_(True) + else: + y = torch.randn( + (batch_size, N), dtype=get_dtype(dtype), device=torch.device("cuda") + ).requires_grad_(True) + + # Make sure tensors are contiguous for TMA kernels + weight_t_contiguous = weight.T.contiguous() + + if provider == "pytorch": + fn = lambda: torch.addmm(y, x, weight.T) # noqa E731 + elif provider == "triton_tma_persistent": + fn = lambda: triton_addmm_fwd_tma_persistent( + x, weight_t_contiguous, y, warp_specialize=False + ) # noqa E731 + elif provider == "triton_tma_persistent_ws": + fn = lambda: triton_addmm_fwd_tma_persistent( + x, weight_t_contiguous, y, warp_specialize=True + ) # noqa E731 + elif provider == "triton_tma_persistent_ws_tlx": + fn = lambda: triton_addmm_fwd_tma_ws_persistent_tlx( + x, weight_t_contiguous, y + ) # noqa E731 + elif provider == "triton_tma_ws_tlx": + fn = lambda: triton_addmm_fwd_tma_ws_tlx(x, weight_t_contiguous, y) # noqa E731 + elif provider == "triton": + fn = lambda: triton_addmm_fwd(x, weight_t_contiguous, y) # noqa E731 + else: + raise ValueError(f"Unknown provider: {provider}") + time.sleep(2) + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + + df = bench_addmm.run(print_data=True, return_df=return_result) + + if return_result: + return configs, df + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_bench.py new file mode 100644 index 000000000..dcfb9819e --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_bench.py @@ -0,0 +1,199 @@ +# pyre-strict +import math +from typing import List, Optional, Tuple + +import click +import pandas as pd +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_jagged import triton_jagged_dense_bmm + +# buck2 run @mode/{opt,inplace} //generative_recommenders/ops/benchmarks:jagged_dense_bmm_bench -- --fwd-only + + +def jagged_dense_bmm( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Wrapper function for jagged_dense_bmm with kernel selection. + Computing out = jagged x dense + jagged has shape (sum_B(M_i), K), dense has shape (B, K, N) + out has shape (sum_B(M_i), N) + """ + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_bmm( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + ) + elif kernel == HammerKernel.PYTORCH: + # PyTorch implementation - manual implementation using standard operations + B, K, N = dense.shape + outputs = [] + for i in range(B): + start_idx = seq_offsets[i] + end_idx = seq_offsets[i + 1] + if start_idx < end_idx: + jagged_seq = jagged[start_idx:end_idx] # (seq_len, K) + dense_seq = dense[i] # (K, N) + output_seq = torch.mm(jagged_seq, dense_seq) # (seq_len, N) + outputs.append(output_seq) + return ( + torch.cat(outputs, dim=0) + if outputs + else torch.empty(0, N, device=jagged.device, dtype=jagged.dtype) + ) + else: + raise ValueError(f"Unsupported kernel: {kernel}") + + +@click.command() +@click.option( + "--batch-size", + type=int, + default=512, +) +@click.option( + "--max-seq-len", + type=int, + default=8192, + show_default=True, +) +@click.option( + "-d", + type=int, + default=64, + show_default=True, +) +@click.option( + "-k", + type=int, + default=64, + show_default=True, +) +@click.option("--dtype", type=str, default="bf16") +@click.option("--fwd-only", is_flag=True) +@click.option("--return-result", type=bool, default=False) +def main( + batch_size: int, + max_seq_len: int, + d: int, + k: int, + dtype: str, + fwd_only: bool, + return_result: bool, +) -> Optional[Tuple[List[triton.testing.Benchmark], List[pd.DataFrame]]]: + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + max_seq_len_log2 = int(round(math.log2(max_seq_len))) + if dtype == "fp32": + pt_dtype = torch.float32 + elif dtype == "fp16": + pt_dtype = torch.float16 + elif dtype == "bf16": + pt_dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {dtype}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["seq_len"], + x_vals=[2**i for i in range(5, max_seq_len_log2 + 1)], + line_arg="provider", + line_vals=["triton", "pytorch"], + line_names=["Triton", "Pytorch"], + styles=[("red", "-"), ("blue", "-")], + ylabel="ms", + plot_name=f"jagged_dense_bmm-b{batch_size}-D{d}-K{k}-{dtype}", + args={ + "batch_size": batch_size, + "D": d, + "K": k, + "dtype": pt_dtype, + "mode": mode, + }, + ) + for mode in (["fwd"] if fwd_only else ["fwd", "bwd"]) + ] + + @triton.testing.perf_report(configs) + def bench_jagged_dense_bmm( + batch_size: int, + seq_len: int, + D: int, + K: int, + mode: str, + provider: str, + dtype: torch.dtype, + ) -> float: + assert mode in ["fwd", "bwd"] + warmup = 25 + rep = 100 + + max_seq_len = seq_len + lengths = torch.randint( + max_seq_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + jagged_size = int(seq_offsets[-1].item()) + jagged = ( + torch.empty((jagged_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + dense = ( + torch.empty((batch_size, D, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if provider == "triton": + fn = lambda: jagged_dense_bmm( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.TRITON, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + elif provider == "pytorch": + fn = lambda: jagged_dense_bmm( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.PYTORCH, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + else: + raise ValueError(f"unsupported provider: {provider}") + + df = bench_jagged_dense_bmm.run(print_data=True, return_df=return_result) + + if return_result: + return configs, df + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_broadcast_add_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_broadcast_add_bench.py new file mode 100644 index 000000000..193704bf9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_bmm_broadcast_add_bench.py @@ -0,0 +1,270 @@ +# pyre-strict +import math +import pickle +from typing import List + +import click +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.jagged_tensors import jagged_dense_bmm_broadcast_add +from generative_recommenders.ops.triton.triton_jagged import ( + jagged_dense_bmm_broadcast_add_kernel, + triton_jagged_dense_bmm, + triton_jagged_dense_broadcast_add, +) + +# buck2 run @mode/{opt,inplace} //generative_recommenders/ops/benchmarks:jagged_dense_bmm_broadcast_add_bench -- --fwd-only + +# To dump the jagged_dense_bmm_broadcast_add_kernel cache +# buck2 run @mode/opt //generative_recommenders/ops/benchmarks:jagged_dense_bmm_broadcast_add_bench -- --fwd-only --dump-cache-dir=/home/${USER}/fbsource/fbcode/generative_recommenders/ops/triton/jagged_dense_bmm_broadcast_add_kernel_cache.pkl + + +def get_kernel(provider: str) -> HammerKernel: + if provider == "triton": + return HammerKernel.TRITON + elif provider == "pytorch": + return HammerKernel.PYTORCH + else: + raise ValueError(f"Unknown provider {provider}") + + +def jagged_dense_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Wrapper function for jagged_dense_broadcast_add with kernel selection. + Computing out = jagged + dense (broadcasted) + jagged has shape (sum_B(M_i), N), dense has shape (B, N) + out has shape (sum_B(M_i), N) + """ + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + ) + elif kernel == HammerKernel.PYTORCH: + # PyTorch implementation - manual implementation using standard operations + B, N = dense.shape + outputs = [] + for i in range(B): + start_idx = seq_offsets[i] + end_idx = seq_offsets[i + 1] + if start_idx < end_idx: + jagged_seq = jagged[start_idx:end_idx] # (seq_len, N) + dense_seq = dense[i] # (N,) + output_seq = jagged_seq + dense_seq # (seq_len, N) + outputs.append(output_seq) + return ( + torch.cat(outputs, dim=0) + if outputs + else torch.empty(0, N, device=jagged.device, dtype=jagged.dtype) + ) + else: + raise ValueError(f"Unsupported kernel: {kernel}") + + +def jagged_dense_bmm( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Wrapper function for jagged_dense_bmm with kernel selection. + Computing out = jagged x dense + jagged has shape (sum_B(M_i), K), dense has shape (B, K, N) + out has shape (sum_B(M_i), N) + """ + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_bmm( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + ) + elif kernel == HammerKernel.PYTORCH: + # PyTorch implementation - manual implementation using standard operations + B, K, N = dense.shape + outputs = [] + for i in range(B): + start_idx = seq_offsets[i] + end_idx = seq_offsets[i + 1] + if start_idx < end_idx: + jagged_seq = jagged[start_idx:end_idx] # (seq_len, K) + dense_seq = dense[i] # (K, N) + output_seq = torch.mm(jagged_seq, dense_seq) # (seq_len, N) + outputs.append(output_seq) + return ( + torch.cat(outputs, dim=0) + if outputs + else torch.empty(0, N, device=jagged.device, dtype=jagged.dtype) + ) + else: + raise ValueError(f"Unsupported kernel: {kernel}") + + +@click.command() +@click.option( + "--batch-size", + type=int, + default=384, +) +@click.option( + "--max-seq-len", + type=int, + default=4096, + show_default=True, +) +@click.option( + "-d", + type=int, + default=512, + show_default=True, +) +@click.option( + "-k", + type=int, + default=512, + show_default=True, +) +@click.option("--dtype", type=str, default="bf16") +@click.option("--fwd-only", is_flag=True) +@click.option("--dump-cache-dir", type=str, default="") +def main( + batch_size: int, + max_seq_len: int, + d: int, + k: int, + dtype: str, + fwd_only: bool, + dump_cache_dir: str, +) -> None: + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + max_seq_len_log2 = int(round(math.log2(max_seq_len))) + if dtype == "fp32": + pt_dtype = torch.float32 + elif dtype == "fp16": + pt_dtype = torch.float16 + elif dtype == "bf16": + pt_dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {dtype}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["seq_len"], + x_vals=[2**i for i in range(8, max_seq_len_log2 + 1)], + line_arg="provider", + line_vals=["triton", "pytorch", "triton_nonfused"], + line_names=["Triton", "Pytorch", "Triton_Nonfused"], + styles=[("red", "-"), ("blue", "-"), ("green", "-")], + ylabel="ms", + plot_name=f"jagged_dense_bmm_broadcast_add-{mode}-b{batch_size}-D{d}-K{k}-{dtype}", + args={ + "batch_size": batch_size, + "D": d, + "K": k, + "dtype": pt_dtype, + "mode": mode, + }, + ) + for mode in (["fwd"] if fwd_only else ["fwd", "bwd"]) + ] + + @triton.testing.perf_report(configs) + def bench_jagged_dense_bmm_broadcast_add( + batch_size: int, + seq_len: int, + D: int, + K: int, + mode: str, + provider: str, + dtype: torch.dtype, + ) -> float: + assert mode in ["fwd", "bwd"] + warmup = 25 + rep = 100 + + max_seq_len = seq_len + lengths = torch.randint( + max_seq_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + jagged_size = int(seq_offsets[-1].item()) + jagged = ( + torch.empty((jagged_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + dense = ( + torch.empty((batch_size, D, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + bias = ( + torch.empty((batch_size, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if provider in ["triton", "pytorch"]: + fn = lambda: jagged_dense_bmm_broadcast_add( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + kernel=get_kernel(provider), + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + elif provider == "triton_nonfused": + fn = lambda: jagged_dense_broadcast_add( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged_dense_bmm( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.TRITON, + ), + dense=bias, + kernel=HammerKernel.TRITON, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + else: + raise ValueError(f"unsupported provider: {provider}") + + bench_jagged_dense_bmm_broadcast_add.run(print_data=True) + if dump_cache_dir: + with open(dump_cache_dir, "wb") as data: + # @lint-ignore PYTHONPICKLEISBAD + pickle.dump(jagged_dense_bmm_broadcast_add_kernel.cache, data) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_broadcast_add_bench.py b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_broadcast_add_bench.py new file mode 100644 index 000000000..049258e3d --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/benchmarks/jagged_dense_broadcast_add_bench.py @@ -0,0 +1,205 @@ +# pyre-strict +import math +import pickle +from typing import List, Optional, Tuple + +import click +import pandas as pd +import torch + +# @manual=//triton:triton +import triton +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_jagged import ( + jagged_dense_broadcast_add_kernel, + triton_jagged_dense_broadcast_add, +) + +# buck2 run @mode/{opt,inplace} //generative_recommenders/ops/benchmarks:jagged_dense_broadcast_add_bench + + +# To dump the jagged_dense_broadcast_add_kernel cache, run: +# buck2 run @mode/{opt,inplace} //generative_recommenders/ops/benchmarks:jagged_dense_broadcast_add_bench -- --dump-ragged-tuner-cache-dir=/home/${USER}/fbsource/fbcode/generative_recommenders/ops/triton/jagged_dense_broadcast_add_kernel_cache.pkl + + +def jagged_dense_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Wrapper function for jagged_dense_broadcast_add with kernel selection. + Computing out = jagged + dense (broadcasted) + jagged has shape (sum_B(M_i), N), dense has shape (B, N) + out has shape (sum_B(M_i), N) + """ + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + ) + elif kernel == HammerKernel.PYTORCH: + # PyTorch implementation - manual implementation using standard operations + B, N = dense.shape + outputs = [] + for i in range(B): + start_idx = seq_offsets[i] + end_idx = seq_offsets[i + 1] + if start_idx < end_idx: + jagged_seq = jagged[start_idx:end_idx] # (seq_len, N) + dense_seq = dense[i] # (N,) + output_seq = jagged_seq + dense_seq # (seq_len, N) + outputs.append(output_seq) + return ( + torch.cat(outputs, dim=0) + if outputs + else torch.empty(0, N, device=jagged.device, dtype=jagged.dtype) + ) + else: + raise ValueError(f"Unsupported kernel: {kernel}") + + +@click.command() +@click.option( + "--batch-size", + type=int, + default=512, +) +@click.option( + "--max-seq-len", + type=int, + default=8192, + show_default=True, +) +@click.option( + "-d", + type=int, + default=64, + show_default=True, +) +@click.option("--dtype", type=str, default="fp32") +@click.option("--fwd-only", is_flag=True) +@click.option("--dump-ragged-tuner-cache-dir", type=str, default="") +@click.option("--return-result", type=bool, default=False) +def main( + batch_size: int, + max_seq_len: int, + d: int, + dtype: str, + fwd_only: bool, + dump_ragged_tuner_cache_dir: str, + return_result: bool, +) -> Optional[Tuple[List[triton.testing.Benchmark], List[pd.DataFrame]]]: + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + max_seq_len_log2 = int(round(math.log2(max_seq_len))) + if dtype == "fp32": + pt_dtype = torch.float32 + elif dtype == "fp16": + pt_dtype = torch.float16 + elif dtype == "bf16": + pt_dtype = torch.bfloat16 + else: + raise ValueError(f"Unsupported data type: {dtype}.") + + configs: List[triton.testing.Benchmark] = [ + triton.testing.Benchmark( + x_names=["seq_len"], + x_vals=[2**i for i in range(5, max_seq_len_log2 + 1)], + line_arg="provider", + line_vals=["triton", "pytorch"], + line_names=["Triton", "Pytorch"], + styles=[("red", "-"), ("blue", "-")], + ylabel="ms", + plot_name=f"jagged_dense_broadcast_add-b{batch_size}-D{d}-{dtype}-{mode}", + args={ + "batch_size": batch_size, + "D": d, + "dtype": pt_dtype, + "mode": mode, + }, + ) + for mode in (["fwd"] if fwd_only else ["fwd", "fwd+bwd"]) + ] + + @triton.testing.perf_report(configs) + def bench_jagged_dense_broadcast_add( + batch_size: int, + seq_len: int, + D: int, + mode: str, + provider: str, + dtype: torch.dtype, + ) -> float: + assert mode in ["fwd", "bwd", "fwd+bwd"] + warmup = 25 + rep = 100 + + max_seq_len = seq_len + lengths = torch.randint( + max_seq_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + jagged_size = int(seq_offsets[-1].item()) + jagged = ( + torch.empty((jagged_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + dense = ( + torch.empty((batch_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if provider == "triton": + fn = lambda: jagged_dense_broadcast_add( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.TRITON, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + elif provider == "pytorch": + fn = lambda: jagged_dense_broadcast_add( # noqa E731 + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + kernel=HammerKernel.PYTORCH, + ) + if mode == "bwd": + o = fn() + do = torch.randn_like(o) + fn = lambda: o.backward(do, retain_graph=True) # noqa E731 + ms = triton.testing.do_bench(fn, warmup=warmup, rep=rep) + return ms + else: + raise ValueError(f"unsupported provider: {provider}") + + df = bench_jagged_dense_broadcast_add.run(print_data=True, return_df=return_result) + + if dump_ragged_tuner_cache_dir: + with open(dump_ragged_tuner_cache_dir, "wb") as data: + # @lint-ignore PYTHONPICKLEISBAD + pickle.dump(jagged_dense_broadcast_add_kernel.cache, data) + + if return_result: + return configs, df + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/generative_recommenders/ops/hstu_attention.py b/recommendation_v4/generative_recommenders/ops/hstu_attention.py new file mode 100644 index 000000000..137482227 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/hstu_attention.py @@ -0,0 +1,353 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.common import HammerKernel, switch_to_contiguous_if_needed +from generative_recommenders.ops.pytorch.pt_hstu_attention import ( + pytorch_cached_hstu_mha, + pytorch_hstu_mha, +) +from generative_recommenders.ops.triton.triton_hstu_attention import ( + triton_cached_hstu_mha, + triton_hstu_mha, +) + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_ragged_hstu_attention + from generative_recommenders.ops.triton_aot.triton_ragged_hstu_attention import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_cached_hstu_mha, + aot_triton_kernel_wrapper_ragged_hstu_mha, + ) +except ImportError: + + def aot_triton_kernel_wrapper_cached_hstu_mha( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE cached_hstu_mha kernel." + ) + + def aot_triton_kernel_wrapper_ragged_hstu_mha( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE ragged_hstu_mha kernel." + ) + + +try: + from hammer.ops.triton.cc.hstu_attention.triton_cc_hstu_attention import ( + triton_cc_hstu_mha, + ) + from hammer.v2.ops.triton.template.tlx_bw_hstu_attention import ( + tlx_bw_hstu_mha_wrapper, + ) +except ImportError: + tlx_bw_hstu_mha_wrapper = None + from generative_recommenders.ops.triton.triton_hstu_attention import ( + triton_hstu_mha as triton_cc_hstu_mha, + ) +from torch.fx._symbolic_trace import is_fx_tracing + +torch.fx.wrap("triton_hstu_mha") +torch.fx.wrap("triton_cached_hstu_mha") + + +@torch.fx.wrap +def hstu_mha_cuda( + max_seq_len: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, +) -> torch.Tensor: + """TorchScript-friendly inference forwarder onto ``torch.ops.hstu.hstu_mha``. + + Bypasses the ``HammerKernel`` enum dispatch in :func:`hstu_mha` so the + scripted graph has a single concrete C++ op to call. Mirrors the + inference-only path of + :func:`generative_recommenders.ops.cpp.cuda_hstu_attention.cuda_hstu_mha_inference_wrapper` + with the subset of arguments :class:`STULayer` actually uses. + """ + return torch.ops.hstu.hstu_mha( + max_seq_len, + alpha, + q, + k, + v, + seq_offsets, + True, # causal + num_targets, + None, # attn_scale + max_attn_len, + 0, # min_full_attn_seq_len + contextual_seq_len, + None, # q_descale + None, # k_descale + None, # v_descale + False, # sort_by_length + False, # deterministic + 0, # sm_margin + 0, # max_q_len + None, # seq_offsets_q + 0, # num_softmax_heads + False, # training + None, # max_seq_len_tensor + None, # contextual_seq_len_tensor + None, # max_attn_len_tensor + None, # min_full_attn_seq_len_tensor + 1, # num_groups + ) + + +def hstu_mha( + max_seq_len: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + causal: bool = True, + dropout_pr: float = 0.0, + training: bool = True, + num_targets: Optional[torch.Tensor] = None, + attn_scale: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + min_full_attn_seq_len: int = 0, + sort_by_length: bool = False, + kernel: HammerKernel = HammerKernel.PYTORCH, + enable_tma: bool = False, +) -> torch.Tensor: + _, H, _ = q.shape + if not is_fx_tracing(): + torch._assert(max_seq_len > 0, "max_seq_len must be larger than 0") + torch._assert(q.dim() == 3, "q must be 3-D") + torch._assert(k.shape == q.shape, "k must be the same shape as q") + torch._assert(v.dim() == 3, "v must be 3-D") + torch._assert(v.shape[0] == q.shape[0], "wrong v shape[0]") + torch._assert(v.shape[1] == H, "wrong v shape[1]") + torch._assert(causal, "only support causal attention") + + if kernel in [ + HammerKernel.TRITON, + HammerKernel.TLX, + HammerKernel.TRITON_CC, + HammerKernel.TRITON_INFERENCE, + ]: + if not is_fx_tracing() and kernel == HammerKernel.TRITON: + torch._assert(q.is_cuda, "q must be CUDA tensor") + torch._assert(k.is_cuda, "k must be CUDA tensor") + torch._assert(v.is_cuda, "v must be CUDA tensor") + torch._assert(seq_offsets.is_cuda, "seq_offsets must be CUDA tensor") + torch._assert(dropout_pr < 1e-6, "dropout for triton path not implemented") + torch._assert( + min_full_attn_seq_len == 0, "min_full_attn_seq_len not implemented" + ) + assert attn_scale is None, "attn_scale not implemented" + q = switch_to_contiguous_if_needed(q) + k = switch_to_contiguous_if_needed(k) + v = switch_to_contiguous_if_needed(v) + seq_offsets = seq_offsets.contiguous() + + if kernel == HammerKernel.TRITON: + return triton_hstu_mha( + N=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length=sort_by_length, + enable_tma=enable_tma, + ) + elif kernel == HammerKernel.TLX: + if tlx_bw_hstu_mha_wrapper is None: + raise ImportError( + "hammer.v2 is required for the TLX kernel. " + "Falling back to TRITON or PYTORCH kernel instead." + ) + return tlx_bw_hstu_mha_wrapper( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + attn_scale=torch.tensor(1.0 / max_seq_len, device=q.device), + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length=sort_by_length, + ) + elif kernel == HammerKernel.TRITON_CC: + return triton_cc_hstu_mha( + N=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_ragged_hstu_mha( + N=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + invalid_attn_mask_type="causal", + num_targets=num_targets, + attn_scale=attn_scale, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + full_attn_size=min_full_attn_seq_len, + num_softmax_heads=0, + ) + else: + return pytorch_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=True, + dropout_pr=dropout_pr, + training=training, + num_targets=num_targets, + attn_scale=attn_scale, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + min_full_attn_seq_len=min_full_attn_seq_len, + ) + + +def delta_hstu_mha( + max_seq_len: int, + alpha: float, + delta_q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + kernel: HammerKernel = HammerKernel.PYTORCH, + enable_tma: bool = False, +) -> torch.Tensor: + L, H, D = delta_q.shape + B = seq_offsets.size(0) - 1 + DeltaSize = L // B + if not is_fx_tracing(): + torch._assert(max_seq_len > 0, "max_seq_len must be larger than 0") + torch._assert(delta_q.dim() == 3, "delta_q must be 3-D") + torch._assert(L % B == 0, "delta_q must be padded") + torch._assert(k.dim() == 3, "k must be 3-D") + torch._assert(k.shape[1] == H, "wrong k shape[1]") + torch._assert(k.shape[2] == D, "wrong k shape[2]") + torch._assert(v.dim() == 3, "v must be 3-D") + torch._assert(v.shape[1] == H, "wrong v shape[1]") + if kernel in [ + HammerKernel.TRITON, + HammerKernel.TRITON_CC, + HammerKernel.TRITON_INFERENCE, + ]: + if not is_fx_tracing() and kernel == HammerKernel.TRITON: + torch._assert(delta_q.is_cuda, "q must be CUDA tensor") + torch._assert(seq_offsets.is_cuda, "seq_offsets must be CUDA tensor") + if num_targets is not None: + torch._assert(num_targets.is_cuda, "num_targets must be CUDA tensor") + seq_offsets = seq_offsets.contiguous() + delta_q = switch_to_contiguous_if_needed(delta_q) + k = switch_to_contiguous_if_needed(k) + v = switch_to_contiguous_if_needed(v) + + if kernel == HammerKernel.TRITON: + return triton_cached_hstu_mha( + N=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + enable_tma=enable_tma, + ) + elif kernel == HammerKernel.TRITON_CC: + return triton_cc_hstu_mha( + N=max_seq_len, + alpha=alpha, + q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + is_delta_q=True, + delta_size=DeltaSize, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + delta_x_offsets = torch.arange( + 0, + L + 1, + DeltaSize, + device=delta_q.device, + dtype=seq_offsets.dtype, + ) + return aot_triton_kernel_wrapper_cached_hstu_mha( + N=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + delta_x_offsets=delta_x_offsets, + seq_offsets=seq_offsets, + num_targets=num_targets, + attn_scale=None, + max_attn_len=max_attn_len, + full_attn_size=0, + ) + else: + return pytorch_cached_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ) diff --git a/recommendation_v4/generative_recommenders/ops/hstu_compute.py b/recommendation_v4/generative_recommenders/ops/hstu_compute.py new file mode 100644 index 000000000..7728c1454 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/hstu_compute.py @@ -0,0 +1,390 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F +from generative_recommenders.ops.layer_norm import layer_norm +from generative_recommenders.ops.mm import addmm +from generative_recommenders.ops.pytorch.pt_hstu_linear import ( + pytorch_hstu_compute_output, +) + +try: + from hammer.ops.triton.cc.addmm.triton_cc_addmm import triton_cc_addmm + from hammer.ops.triton.cc.group_norm_mul_dropout.triton_cc_group_norm_mul_dropout import ( + triton_cc_group_norm_mul_dropout_wrapper, + ) + from hammer.ops.triton.cc.layer_norm_mul_dropout.triton_cc_layer_norm_mul_dropout import ( + triton_cc_layer_norm_mul_dropout_wrapper, + ) +except ImportError: + triton_cc_addmm = None + triton_cc_group_norm_mul_dropout_wrapper = None + triton_cc_layer_norm_mul_dropout_wrapper = None +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.hstu_attention import hstu_mha, hstu_mha_cuda +from generative_recommenders.ops.triton.triton_hstu_linear import ( + triton_hstu_compute_output, +) +from generative_recommenders.ops.triton.triton_hstu_preprocess_and_attention import ( + triton_hstu_preprocess_and_attention, +) +from torch.fx._symbolic_trace import is_fx_tracing + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_group_norm_mul_dropout + from generative_recommenders.ops.triton_aot.triton_group_norm_mul_dropout import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_group_norm_mul_dropout, + ) + + # @manual=//generative_recommenders/ops/triton_aot:triton_layer_norm_mul_dropout + from generative_recommenders.ops.triton_aot.triton_layer_norm_mul_dropout import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_layer_norm_mul_dropout, + ) +except ImportError: + + def aot_triton_kernel_wrapper_group_norm_mul_dropout( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE group_norm_mul_dropout kernel." + ) + + def aot_triton_kernel_wrapper_layer_norm_mul_dropout( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE layer_norm_mul_dropout kernel." + ) + + +torch.fx.wrap("triton_hstu_compute_output") + + +def hstu_compute_uqvk( + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + if torch.jit.is_scripting(): + # Script-mode fast path: pure PyTorch, no HammerKernel dispatch. + normed_x = F.layer_norm( + x, + normalized_shape=(x.shape[-1],), + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + ) + uvqk = torch.addmm(uvqk_bias, normed_x, uvqk_weight) + else: + normed_x = layer_norm( + x, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + kernel=kernel, + ) + # NOTE: for AMD training, we go with torch.addmm instead of the triton + # version before Triton on AMD achieves on-par perf with NV GPU. + if torch.version.hip and kernel == HammerKernel.TRITON: + uvqk = torch.addmm(uvqk_bias, normed_x, uvqk_weight) + else: + uvqk = addmm(uvqk_bias, normed_x, uvqk_weight, kernel) + u, v, q, k = torch.split( + uvqk, + [ + hidden_dim * num_heads, + hidden_dim * num_heads, + attn_dim * num_heads, + attn_dim * num_heads, + ], + dim=1, + ) + u = F.silu(u) + q = q.view(-1, num_heads, attn_dim) + k = k.view(-1, num_heads, attn_dim) + v = v.view(-1, num_heads, hidden_dim) + return u, q, k, v + + +def hstu_compute_output( + attn: torch.Tensor, + u: torch.Tensor, + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + output_weight: torch.Tensor, + num_heads: int, + linear_dim: int, + dropout_ratio: float, + training: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + group_norm: bool, + recompute_y_in_backward: bool, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + return pytorch_hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + output_weight=output_weight, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=linear_dim, + ) + if kernel == HammerKernel.TRITON: + return triton_hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + output_weight=output_weight, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=linear_dim, + seed=None, + recompute_y_in_backward=recompute_y_in_backward, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + if group_norm: + y = aot_triton_kernel_wrapper_group_norm_mul_dropout( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + silu_u=mul_u_activation_type == "silu", + concat_ux=concat_u and concat_x, + num_heads=num_heads, + linear_dim=linear_dim, + ) + else: + y = aot_triton_kernel_wrapper_layer_norm_mul_dropout( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=mul_u_activation_type == "silu", + concat_ux=concat_u and concat_x, + mul_u_activation_type=mul_u_activation_type, + ) + return addmm(x, y, output_weight, kernel) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_group_norm_mul_dropout_wrapper is None or triton_cc_addmm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in hstu_compute_output." + ) + if group_norm: + y = triton_cc_group_norm_mul_dropout_wrapper( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_ux=concat_u and concat_x, + num_heads=num_heads, + linear_dim=linear_dim, + ) + else: + y = triton_cc_layer_norm_mul_dropout_wrapper( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + ) + return triton_cc_addmm(x, y, output_weight) + else: + return pytorch_hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + output_weight=output_weight, + eps=norm_eps, + dropout_ratio=dropout_ratio, + training=training, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=linear_dim, + ) + + +def hstu_preprocess_and_attention( + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: torch.Tensor, + max_seq_len: int, + seq_offsets: torch.Tensor, + attn_alpha: float, + causal: bool, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + recompute_uvqk_in_backward: bool, + recompute_normed_x_in_backward: bool, + sort_by_length: bool, + prefill: bool = False, + kernel: HammerKernel = HammerKernel.PYTORCH, + enable_tma: Optional[bool] = None, +) -> Tuple[torch.Tensor, torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + if not is_fx_tracing(): + torch._assert(max_seq_len > 0, "max_seq_len must be larger than 0") + torch._assert(x.dim() == 2, "x must be 2-D") + torch._assert( + x.shape[1] == uvqk_weight.shape[0], + "x.shape[1] must equal uvqk_weight.shape[0]", + ) + torch._assert( + uvqk_weight.shape[1] == 2 * num_heads * (hidden_dim + attn_dim), + "uvqk_weight.shape[1] must equal 2 * num_heads * (hidden_dim + attn_dim)", + ) + torch._assert(causal is True, "only causal attention is supported.") + if torch.jit.is_scripting(): + # Script-mode: compute uvqk via PyTorch fallback then call the + # libtorch-callable CUDA HSTU MHA op directly. Avoids both the + # HammerKernel enum dispatch and the Triton-only fused path. + u, q, k, v = hstu_compute_uqvk( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=num_heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + kernel=HammerKernel.PYTORCH, + ) + attn_output = hstu_mha_cuda( + max_seq_len=max_seq_len, + alpha=attn_alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ).view(-1, hidden_dim * num_heads) + return u, attn_output, k, v + if kernel == HammerKernel.TRITON and prefill is False: + u, attn_output = triton_hstu_preprocess_and_attention( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=num_heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + attn_alpha=attn_alpha, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + recompute_uvqk_in_backward=recompute_uvqk_in_backward, + recompute_normed_x_in_backward=recompute_normed_x_in_backward, + sort_by_length=sort_by_length, + enable_tma=enable_tma, + ) + attn_output = attn_output.view(-1, hidden_dim * num_heads) + k = None + v = None + else: + u, q, k, v = hstu_compute_uqvk( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=num_heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + kernel=kernel, + ) + attn_output = hstu_mha( + max_seq_len=max_seq_len, + alpha=attn_alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=causal, + dropout_pr=0.0, + training=False, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length=sort_by_length, + kernel=kernel, + ).view(-1, hidden_dim * num_heads) + return u, attn_output, k, v diff --git a/recommendation_v4/generative_recommenders/ops/jagged_tensors.py b/recommendation_v4/generative_recommenders/ops/jagged_tensors.py new file mode 100644 index 000000000..73e3c4a73 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/jagged_tensors.py @@ -0,0 +1,451 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.pytorch.pt_jagged import pytorch_jagged_dense_bmm_add +from generative_recommenders.ops.pytorch.pt_jagged_tensors import ( + pytorch_concat_2D_jagged, + pytorch_hstu_concat_l2_embeddings, + pytorch_hstu_split_l2_embeddings, + pytorch_split_2D_jagged, +) +from generative_recommenders.ops.triton.triton_jagged import triton_jagged_dense_bmm_add +from generative_recommenders.ops.triton.triton_jagged_tensors import ( + triton_concat_2D_jagged, + triton_concat_2D_jagged_multirow, + triton_split_2D_jagged, + triton_split_2D_jagged_multirow, +) +from torch.fx._symbolic_trace import is_fx_tracing + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_concat_2d_jagged + from generative_recommenders.ops.triton_aot.triton_concat_2d_jagged import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_concat_2D_jagged, + ) + + # @manual=//generative_recommenders/ops/triton_aot:triton_split_2d_jagged + from generative_recommenders.ops.triton_aot.triton_split_2d_jagged import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_split_2D_jagged, + ) +except ImportError: + + def aot_triton_kernel_wrapper_concat_2D_jagged( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE concat_2D_jagged kernel." + ) + + def aot_triton_kernel_wrapper_split_2D_jagged( + *args: object, + **kwargs: object, + ) -> Tuple[torch.Tensor, torch.Tensor]: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE split_2D_jagged kernel." + ) + + +torch.fx.wrap("triton_jagged_dense_bmm_add") + +try: + from hammer.ops.triton.cc.jagged_dense_bmm.triton_cc_jagged_dense_bmm import ( + triton_cc_jagged_dense_bmm, + ) +except ImportError: + triton_cc_jagged_dense_bmm = None + + +torch.fx.wrap("triton_concat_2D_jagged") +torch.fx.wrap("triton_split_2D_jagged") +torch.fx.wrap("triton_concat_2D_jagged_multirow") +torch.fx.wrap("triton_split_2D_jagged_multirow") + + +def concat_2D_jagged( + max_seq_len: int, + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: Optional[int] = None, + max_len_right: Optional[int] = None, + offsets_left: Optional[torch.Tensor] = None, + offsets_right: Optional[torch.Tensor] = None, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + return pytorch_concat_2D_jagged( + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + if not is_fx_tracing(): + torch._assert(values_left.dim() == 2, "values_left must be 2D") + torch._assert(values_right.dim() == 2, "values_right must be 2D") + torch._assert( + values_right.shape[1] == values_left.shape[1], + f"values_left shape[1] must be equal to values_right shape[1] {values_left.shape[1]} vs {values_right.shape[1]}", + ) + if kernel == HammerKernel.TRITON: + return triton_concat_2D_jagged( + max_seq_len=max_seq_len, + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + aott_values_left = values_left + aott_values_right = values_right + if offsets_left is None: + assert max_len_left is not None + aott_values_left = values_left.reshape( + -1, + max_len_left, + values_left.shape[-1], + ) + if offsets_right is None: + assert max_len_right is not None + aott_values_right = values_right.reshape( + -1, + max_len_right, + values_right.shape[-1], + ) + return aot_triton_kernel_wrapper_concat_2D_jagged( + max_seq_len=max_seq_len, + values_a=aott_values_left, + values_b=aott_values_right, + offsets_a=offsets_left, + offsets_b=offsets_right, + ) + else: + return pytorch_concat_2D_jagged( + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + + +def split_2D_jagged( + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int] = None, + total_len_right: Optional[int] = None, + max_len_left: Optional[int] = None, + max_len_right: Optional[int] = None, + offsets_left: Optional[torch.Tensor] = None, + offsets_right: Optional[torch.Tensor] = None, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> Tuple[torch.Tensor, torch.Tensor]: + if torch.jit.is_scripting(): + return pytorch_split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + if not is_fx_tracing(): + torch._assert(values.dim() == 2, "values must be 2D") + torch._assert( + offsets_left is not None or offsets_right is not None, + "offsets_left and offsets_right cannot be None at the same time", + ) + if offsets_left is None: + torch._assert( + max_len_left is not None, + "max_len_left must be provided when offsets_left is None", + ) + if offsets_right is None: + torch._assert( + max_len_right is not None, + "max_len_right must be provided when offsets_right is None", + ) + if offsets_left is not None and offsets_right is not None: + torch._assert( + offsets_left.shape[0] == offsets_right.shape[0], + "offsets_left shape[0] must be equal to offsets_right shape[0]", + ) + if kernel == HammerKernel.TRITON: + return triton_split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + total_len_left=total_len_left, + total_len_right=total_len_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + dense_size = 0 + if offsets_left is None and max_len_left is not None: + dense_size = max_len_left + elif offsets_right is None and max_len_right is not None: + dense_size = max_len_right + split_left, split_right = aot_triton_kernel_wrapper_split_2D_jagged( + values=values, + max_seq_len=max_seq_len, + offsets_a=offsets_left, + offsets_b=offsets_right, + dense_size=dense_size, + ) + if offsets_left is None: + split_left = split_left.reshape(-1, split_left.shape[-1]) + if offsets_right is None: + split_right = split_right.reshape(-1, split_right.shape[-1]) + return split_left, split_right + else: + return pytorch_split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + + +def hstu_split_l2_embeddings( + max_seq_len: int, + x: torch.Tensor, + prefix_offsets: torch.Tensor, + l2_offsets: torch.Tensor, + contextual_seq_len: int, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> Tuple[torch.Tensor, torch.Tensor]: + if kernel == HammerKernel.TRITON: + return triton_split_2D_jagged( + max_seq_len=max_seq_len, + values=x, + total_len_right=None, + total_len_left=None, + max_len_left=None, + max_len_right=None, + offsets_left=prefix_offsets, + offsets_right=l2_offsets, + n_prefix_to_right=contextual_seq_len, + ) + else: + return pytorch_hstu_split_l2_embeddings( + max_seq_len=max_seq_len, + x=x, + prefix_offsets=prefix_offsets, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + ) + + +def hstu_concat_l2_embeddings( + max_prefix_len: int, + prefix_x: torch.Tensor, + prefix_offsets: torch.Tensor, + max_l2_len: int, + l2_x: torch.Tensor, + l2_offsets: torch.Tensor, + contextual_seq_len: int, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if kernel == HammerKernel.TRITON: + return triton_concat_2D_jagged( + max_seq_len=max_prefix_len + max_l2_len, + values_left=prefix_x, + values_right=l2_x, + max_len_left=max_prefix_len, + max_len_right=max_l2_len, + offsets_left=prefix_offsets, + offsets_right=l2_offsets, + n_prefix_from_right=contextual_seq_len, + ) + else: + return pytorch_hstu_concat_l2_embeddings( + contextual_seq_len=contextual_seq_len, + max_prefix_len=max_prefix_len, + prefix_x=prefix_x, + prefix_offsets=prefix_offsets, + max_l2_len=max_l2_len, + l2_x=l2_x, + l2_offsets=l2_offsets, + ) + + +def jagged_dense_bmm_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + """ + Computing out = jagged x dense + bias + jagged has shape (sum_B(M_i), K), dense has shape (B, K, N), and bias has shape (B, N) + out has shape (sum_B(M_i), N) + """ + if not is_fx_tracing(): + _, K = jagged.shape + B, _, N = dense.shape + torch._assert(dense.shape[1] == K, "wrong dense shape[1]") + torch._assert(seq_offsets.shape[0] == B + 1, "wrong seq_offsets shape[0]") + torch._assert(bias.shape[0] == B, "wrong bias shape[0]") + torch._assert(bias.shape[1] == N, "wrong bias shape[1]") + if kernel == HammerKernel.TRITON: + return triton_jagged_dense_bmm_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + elementwise=False, + ) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_jagged_dense_bmm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in jagged_dense_bmm_broadcast_add." + ) + return triton_cc_jagged_dense_bmm( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + ) + else: + return pytorch_jagged_dense_bmm_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + ) + + +def concat_2D_jagged_multirow( + max_seq_len: int, + values_left: torch.Tensor, + values_right: torch.Tensor, + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + max_len_left: int, + max_len_right: int, + kernel: HammerKernel = HammerKernel.TRITON, +) -> torch.Tensor: + if not is_fx_tracing(): + torch._assert(values_left.dim() == 2, "values_left must be 2D") + torch._assert(values_right.dim() == 2, "values_right must be 2D") + torch._assert( + values_right.shape[1] == values_left.shape[1], + f"values_left shape[1] must be equal to values_right shape[1] {values_left.shape[1]} vs {values_right.shape[1]}", + ) + if offsets_left is not None and offsets_right is not None: + torch._assert( + offsets_left.shape[0] == offsets_right.shape[0], + "offsets_left and offsets_right must have the same batch dimension", + ) + + if kernel == HammerKernel.TRITON: + return triton_concat_2D_jagged_multirow( + max_seq_len=max_seq_len, + values_a=values_left, + values_b=values_right, + offsets_a=offsets_left, + offsets_b=offsets_right, + max_len_a=max_len_left, + max_len_b=max_len_right, + ) + else: + return concat_2D_jagged( + max_seq_len=max_seq_len, + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + kernel=kernel, + ) + + +def split_2D_jagged_multirow( + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int] = None, + total_len_right: Optional[int] = None, + max_len_left: Optional[int] = None, + max_len_right: Optional[int] = None, + offsets_left: Optional[torch.Tensor] = None, + offsets_right: Optional[torch.Tensor] = None, + kernel: HammerKernel = HammerKernel.TRITON, +) -> Tuple[torch.Tensor, torch.Tensor]: + if not is_fx_tracing(): + torch._assert(values.dim() == 2, "values must be 2D") + torch._assert( + offsets_left is not None or offsets_right is not None, + "offsets_left and offsets_right cannot be None at the same time", + ) + if offsets_left is None: + torch._assert( + max_len_left is not None, + "max_len_left must be provided when offsets_left is None", + ) + if offsets_right is None: + torch._assert( + max_len_right is not None, + "max_len_right must be provided when offsets_right is None", + ) + if offsets_left is not None and offsets_right is not None: + torch._assert( + offsets_left.shape[0] == offsets_right.shape[0], + "offsets_left and offsets_right must have the same batch dimension", + ) + + if kernel == HammerKernel.TRITON: + return triton_split_2D_jagged_multirow( + max_seq_len=max_seq_len, + values=values, + total_len_left=total_len_left, + total_len_right=total_len_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + ) + else: + return split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + total_len_left=total_len_left, + total_len_right=total_len_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left, + offsets_right=offsets_right, + kernel=kernel, + ) diff --git a/recommendation_v4/generative_recommenders/ops/layer_norm.py b/recommendation_v4/generative_recommenders/ops/layer_norm.py new file mode 100644 index 000000000..74ed377d6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/layer_norm.py @@ -0,0 +1,330 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + + +from typing import List + +import torch +from generative_recommenders.ops.pytorch.pt_layer_norm import ( + pytorch_layer_norm, + pytorch_rms_norm, + pytorch_swish_layer_norm, +) +from generative_recommenders.ops.triton.triton_layer_norm import triton_rms_norm + +try: + from hammer.ops.triton.cc.rms_norm.triton_cc_rms_norm import triton_cc_rms_norm + from hammer.ops.triton.cc.swish_layer_norm.triton_cc_swish_layer_norm import ( + triton_cc_swish_layer_norm, + ) +except ImportError: + triton_cc_swish_layer_norm = None + triton_cc_rms_norm = None +from generative_recommenders.common import HammerKernel, HammerModule +from generative_recommenders.ops.triton.triton_layer_norm import ( + triton_layer_norm, + triton_swish_layer_norm, +) +from torch.fx._symbolic_trace import is_fx_tracing + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_layer_norm + from generative_recommenders.ops.triton_aot.triton_layer_norm import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_swish_layer_norm, + ) + + # @manual=//generative_recommenders/ops/triton_aot:triton_rms_norm + from generative_recommenders.ops.triton_aot.triton_rms_norm import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_rms_norm, + ) +except ImportError: + + def aot_triton_kernel_wrapper_swish_layer_norm( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + is_swish: bool, + ) -> torch.Tensor: + raise ImportError( + "AOT-T is required for the TRITON_INFERENCE swish_layer_norm kernel." + ) + + def aot_triton_kernel_wrapper_rms_norm( + x: torch.Tensor, + weight: torch.Tensor, + eps: float, + silu: bool, + ) -> torch.Tensor: + raise ImportError("AOT-T is required for the TRITON_INFERENCE rms_norm kernel.") + + +torch.fx.wrap("triton_layer_norm") +torch.fx.wrap("triton_swish_layer_norm") +torch.fx.wrap("triton_rms_norm") + + +def layer_norm( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float = 1e-5, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # Script-mode fast path: bypass the HammerKernel ladder (which would + # drag in is_fx_tracing()'s closed-over global bool). + return torch.nn.functional.layer_norm( + x, + normalized_shape=(x.shape[-1],), + weight=weight, + bias=bias, + eps=eps, + ) + if kernel == HammerKernel.TRITON: + if not is_fx_tracing(): + torch._assert(not x.is_cpu, "x must be device tensor") + torch._assert(not weight.is_cpu, "weight must be device tensor") + torch._assert(not bias.is_cpu, "bias must be device tensor") + return triton_layer_norm(x, weight, bias, eps) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_swish_layer_norm( + x, + weight, + bias, + eps, + is_swish=False, + ) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_swish_layer_norm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in layer_norm." + ) + return triton_cc_swish_layer_norm( + x, + weight, + bias, + eps, + is_swish=False, + ) + else: + return pytorch_layer_norm( + x, + [ + x.shape[-1], + ], + weight, + bias, + eps, + ) + + +def rms_norm( + x: torch.Tensor, + weight: torch.Tensor, + eps: float = 1e-5, + kernel: HammerKernel = HammerKernel.PYTORCH, + silu: bool = False, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # Script-mode fast path: bypass the HammerKernel ladder. + x_f = x.float() + norm = torch.rsqrt(x_f.pow(2).mean(-1, keepdim=True) + eps) + out = (x_f * norm * weight.float()).to(x.dtype) + if silu: + out = torch.nn.functional.silu(out) + return out + if kernel == HammerKernel.TRITON: + if not is_fx_tracing(): + torch._assert(not x.is_cpu, "x must be device tensor") + torch._assert(not weight.is_cpu, "weight must be device tensor") + return triton_rms_norm(x, weight, eps, silu) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_rms_norm(x, weight, eps, silu) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_rms_norm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in rms_norm." + ) + return triton_cc_rms_norm( + x, + weight, + eps, + silu=silu, + ) + else: + return pytorch_rms_norm( + x, + [ + x.shape[-1], + ], + weight, + eps, + silu, + ) + + +def swish_layer_norm( + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float = 1e-5, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # Script-mode fast path: bypass the HammerKernel ladder (which + # otherwise drags in is_fx_tracing(), Triton/Triton_CC closures, + # etc.) and call pure PyTorch directly. + return pytorch_swish_layer_norm( + x, + [x.shape[-1]], + weight, + bias, + eps, + ) + if kernel == HammerKernel.TRITON: + if not is_fx_tracing(): + torch._assert(not x.is_cpu, "x must be device tensor") + torch._assert(not weight.is_cpu, "weight must be device tensor") + torch._assert(not bias.is_cpu, "bias must be device tensor") + return triton_swish_layer_norm(x, [x.shape[-1]], weight, bias, eps) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_swish_layer_norm( + x, + weight, + bias, + eps, + is_swish=True, + ) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_swish_layer_norm is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in swish_layer_norm." + ) + return triton_cc_swish_layer_norm( + x, + weight, + bias, + eps, + is_swish=True, + ) + else: + return pytorch_swish_layer_norm( + x, + [ + x.shape[-1], + ], + weight, + bias, + eps, + ) + + +class LayerNorm(HammerModule): + def __init__( + self, + dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._normalized_shape: List[int] = [dim] + self._eps = eps + self.weight = torch.nn.Parameter( + torch.ones(self._normalized_shape), + ) + self.bias = torch.nn.Parameter( + torch.zeros(self._normalized_shape), + ) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return layer_norm( + x=x, + weight=self.weight, + bias=self.bias, + eps=self._eps, + kernel=self.hammer_kernel(), + ) + + +class RMSNorm(HammerModule): + def __init__( + self, + dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._eps = eps + self.weight = torch.nn.Parameter(torch.ones(dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return rms_norm( + x, + self.weight, + self._eps, + silu=False, + kernel=self.hammer_kernel(), + ) + + +class RMSNormSilu(HammerModule): + def __init__( + self, + dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._eps = eps + self.weight = torch.nn.Parameter(torch.ones(dim)) + + def forward(self, x: torch.Tensor) -> torch.Tensor: + return rms_norm( + x, + self.weight, + self._eps, + silu=True, + kernel=self.hammer_kernel(), + ) + + +class SwishLayerNorm(HammerModule): + def __init__( + self, + dim: int, + eps: float = 1e-5, + is_inference: bool = False, + ) -> None: + super().__init__(is_inference=is_inference) + self._normalized_shape: List[int] = [dim] + self.weight = torch.nn.Parameter(torch.ones(self._normalized_shape)) + self.bias = torch.nn.Parameter(torch.zeros(self._normalized_shape)) + self._eps = eps + + def forward( + self, + x: torch.Tensor, + ) -> torch.Tensor: + return swish_layer_norm( + x=x, + weight=self.weight, + bias=self.bias, + eps=self._eps, + kernel=self.hammer_kernel(), + ) diff --git a/recommendation_v4/generative_recommenders/ops/mm.py b/recommendation_v4/generative_recommenders/ops/mm.py new file mode 100644 index 000000000..31a5c5d36 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/mm.py @@ -0,0 +1,60 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import torch + +try: + from hammer.ops.triton.cc.addmm.triton_cc_addmm import triton_cc_addmm +except ImportError: + triton_cc_addmm = None +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_addmm import triton_addmm + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_addmm + from generative_recommenders.ops.triton_aot.triton_addmm import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_addmm, + ) +except ImportError: + + def aot_triton_kernel_wrapper_addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, + ) -> torch.Tensor: + raise ImportError("AOT-T is required for the TRITON_INFERENCE addmm kernel.") + + +def addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + return torch.addmm(input, mat1, mat2) + if kernel == HammerKernel.TRITON: + return triton_addmm(input, mat1, mat2) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_addmm(input, mat1, mat2) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_addmm is None: + raise ImportError("hammer is required for the TRITON_CC kernel in addmm.") + return triton_cc_addmm(input, mat1, mat2) + else: + return torch.addmm(input, mat1, mat2) diff --git a/recommendation_v4/generative_recommenders/ops/position.py b/recommendation_v4/generative_recommenders/ops/position.py new file mode 100644 index 000000000..e090827e3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/position.py @@ -0,0 +1,147 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.ops.pytorch.pt_position import ( + pytorch_add_timestamp_positional_embeddings, +) + +try: + from hammer.ops.triton.cc.add_timestamp_position_embeddings.triton_cc_add_timestamp_position_embeddings import ( + triton_cc_add_timestamp_position_embeddings, + ) +except ImportError: + triton_cc_add_timestamp_position_embeddings = None +from generative_recommenders.common import HammerKernel +from generative_recommenders.ops.triton.triton_position import ( + triton_add_timestamp_positional_embeddings, +) + +try: + # @manual=//generative_recommenders/ops/triton_aot:triton_position + from generative_recommenders.ops.triton_aot.triton_position import ( # pyre-ignore[21] + aot_triton_kernel_wrapper_position, + ) +except ImportError: + + def aot_triton_kernel_wrapper_position( + *args: object, + **kwargs: object, + ) -> torch.Tensor: + raise ImportError("AOT-T is required for the TRITON_INFERENCE position kernel.") + + +torch.fx.wrap("triton_add_timestamp_positional_embeddings") + + +def add_timestamp_positional_embeddings( + alpha: float, + max_seq_len: int, + max_contextual_seq_len: int, + position_embeddings_weight: torch.Tensor, + timestamp_embeddings_weight: torch.Tensor, + seq_offsets: torch.Tensor, + seq_lengths: torch.Tensor, + seq_embeddings: torch.Tensor, + timestamps: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str = "sqrt", + kernel: HammerKernel = HammerKernel.PYTORCH, +) -> torch.Tensor: + if torch.jit.is_scripting(): + # Script-mode fast path: bypass the HammerKernel ladder. + seq_embeddings = seq_embeddings * alpha + return pytorch_add_timestamp_positional_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + assert time_bucket_fn in ["sqrt", "log"] + seq_embeddings = seq_embeddings * alpha + if kernel == HammerKernel.TRITON: + return triton_add_timestamp_positional_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + elif kernel == HammerKernel.TRITON_INFERENCE: + return aot_triton_kernel_wrapper_position( + alpha=1.0, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + position_embeddings_weight=position_embeddings_weight.to(torch.float32), + timestamp_embeddings_weight=timestamp_embeddings_weight.to(torch.float32), + seq_offsets=seq_offsets, + seq_lengths=seq_lengths, + seq_embeddings=seq_embeddings, + timestamps=timestamps, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + elif kernel == HammerKernel.TRITON_CC: + if triton_cc_add_timestamp_position_embeddings is None: + raise ImportError( + "hammer is required for the TRITON_CC kernel in add_timestamp_positional_embeddings." + ) + return triton_cc_add_timestamp_position_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) + else: + return pytorch_add_timestamp_positional_embeddings( + seq_embeddings=seq_embeddings, + seq_offsets=seq_offsets, + pos_embeddings=position_embeddings_weight, + ts_embeddings=timestamp_embeddings_weight, + timestamps=timestamps, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + ) diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py new file mode 100644 index 000000000..32575c4db --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_attention.py @@ -0,0 +1,251 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +import torch.nn.functional as F + + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +@torch.fx.wrap +def _get_valid_attn_mask( + device: torch.device, + causal: bool, + N: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + min_full_attn_seq_len: int = 0, +) -> torch.Tensor: + ids = torch.arange(0, N, device=device).view(1, N) + max_ids = seq_lengths.view(-1, 1, 1) + if contextual_seq_len > 0: + ids = ids - contextual_seq_len + 1 + ids = torch.clamp(ids, min=0) + max_ids = max_ids - contextual_seq_len + 1 + if num_targets is not None: + max_ids = max_ids - num_targets.view(-1, 1, 1) + ids = torch.clamp( + ids, + max=max_ids, + ) + row_ids = ids.view(-1, N, 1).expand(-1, N, N) + col_ids = ids.view(-1, 1, N).expand(-1, N, N) + else: + row_ids = ids.view(N, 1).expand(N, N) + col_ids = row_ids.t() + row_ids = row_ids.view(1, N, N) + col_ids = col_ids.view(1, N, N) + row_col_dist = row_ids - col_ids + valid_attn_mask = torch.eye(N, device=device, dtype=torch.bool).view(1, N, N) + if not causal: + row_col_dist = torch.where(row_col_dist > 0, row_col_dist, -row_col_dist) + valid_attn_mask = torch.logical_or(valid_attn_mask, row_col_dist > 0) + if max_attn_len > 0: + if min_full_attn_seq_len > 0: + valid_attn_mask = torch.logical_and( + valid_attn_mask, + torch.logical_or( + row_col_dist <= max_attn_len, + row_ids >= max_ids - min_full_attn_seq_len, + ), + ) + else: + valid_attn_mask = torch.logical_and( + valid_attn_mask, row_col_dist <= max_attn_len + ) + if contextual_seq_len > 0: + valid_attn_mask = torch.logical_or( + valid_attn_mask, torch.logical_and(row_ids == 0, col_ids < max_ids) + ) + return valid_attn_mask + + +def _pad_qkv( + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + N: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + L, H, D = q.shape + V = v.shape[2] + padded_q = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=q.reshape(L, H * D), + offsets=[seq_offsets], + max_lengths=[N], + padding_value=0.0, + ) + .view(-1, N, H, D) + .transpose(1, 2) + ) # [B, H, N, A] + padded_k = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=k.reshape(L, H * D), + offsets=[seq_offsets], + max_lengths=[N], + padding_value=0.0, + ) + .view(-1, N, H, D) + .transpose(1, 2) + ) # [B, H, N, A] + padded_v = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=v.reshape(L, H * V), + offsets=[seq_offsets], + max_lengths=[N], + padding_value=0.0, + ) + .view(-1, N, H, V) + .transpose(1, 2) + ) # [B, H, N, D] + return padded_q, padded_k, padded_v + + +@torch.fx.wrap +def pytorch_hstu_mha( + max_seq_len: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + causal: bool = True, + dropout_pr: float = 0.0, + training: bool = True, + num_targets: Optional[torch.Tensor] = None, + attn_scale: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + min_full_attn_seq_len: int = 0, +) -> torch.Tensor: + L, H, _ = q.shape + V = v.shape[2] + q, k, v = _pad_qkv( + q, k, v, seq_offsets, max_seq_len + ) # [B, H, N, D) and [B, H, N, V] + qk_attn = torch.einsum("bhxa,bhya->bhxy", q, k) * alpha + if attn_scale is not None: + if attn_scale.ndim > 0: + attn_scale = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=attn_scale.unsqueeze(-1), + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + .unsqueeze(1) + .to(qk_attn.dtype) + ) + + qk_attn = F.silu(qk_attn) * attn_scale + else: + qk_attn = F.silu(qk_attn) / max_seq_len + valid_attn_mask = _get_valid_attn_mask( + device=q.device, + causal=causal, + N=max_seq_len, + seq_lengths=seq_offsets[1:] - seq_offsets[:-1], + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + min_full_attn_seq_len=min_full_attn_seq_len, + ) + # raise NotImplementedError(valid_attn_mask[0, :, :].to(torch.int32)) + qk_attn = qk_attn * valid_attn_mask.unsqueeze(1) + if dropout_pr > 0.0: + qk_attn = F.dropout(qk_attn, p=dropout_pr, training=training) + attn_dense = torch.einsum("bhxd,bhdv->bhxv", qk_attn, v) # [B, H, N, V] + return torch.ops.fbgemm.dense_to_jagged( + attn_dense.transpose(1, 2).flatten(2, 3), # [B, N, H, V]->[B, N, H * V] + [seq_offsets], + L, + )[0].view(L, H, V) + + +@torch.fx.wrap +def pytorch_cached_hstu_mha( + max_seq_len: int, + alpha: float, + delta_q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, +) -> torch.Tensor: + L, H, D = delta_q.shape + _, _, V = v.shape + B = seq_offsets.size(0) - 1 + delta_size = L // B + delta_q = delta_q.view(B, -1, H, D).transpose(1, 2) + full_k = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=k.reshape(-1, H * D), + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + .view(B, -1, H, D) + .transpose(1, 2) + ) + full_v = ( + torch.ops.fbgemm.jagged_to_padded_dense( + values=v.reshape(-1, H * V), + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + .view(B, -1, H, V) + .transpose(1, 2) + ) + qk_attn = torch.einsum("bhxa,bhya->bhxy", delta_q, full_k) * alpha + qk_attn = F.silu(qk_attn) / max_seq_len + full_valid_attn_mask = _get_valid_attn_mask( + device=delta_q.device, + causal=True, + N=max_seq_len, + seq_lengths=seq_offsets[1:] - seq_offsets[:-1], + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ) + seq_lengths = seq_offsets[1:] - seq_offsets[:-1] + mask = torch.arange(max_seq_len, device=delta_q.device).view(1, -1) + mask = torch.logical_and( + mask >= (seq_lengths - delta_size).view(-1, 1), + mask < seq_lengths.view(-1, 1), + ) + valid_attn_mask = ( + full_valid_attn_mask.expand(B, -1, -1) + .flatten(0, 1)[mask.view(-1), :] + .view(-1, delta_size, max_seq_len) + ) + qk_attn = qk_attn * valid_attn_mask.unsqueeze(1) + attn_output = torch.einsum("bhxd,bhdv->bhxv", qk_attn, full_v) + return attn_output.transpose(1, 2).reshape(-1, H, V) diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_linear.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_linear.py new file mode 100644 index 000000000..6ea94a565 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_hstu_linear.py @@ -0,0 +1,130 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import torch +import torch.nn.functional as F + + +def pytorch_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, +) -> torch.Tensor: + dtype = x.dtype + x = x.to(torch.float32) + u = u.to(torch.float32) + if group_norm: + if silu_u: + u = F.silu(u) + u = u.to(torch.float32) + y = u * F.group_norm( + x.view(-1, num_heads, linear_dim), + num_groups=num_heads, + weight=weight.to(torch.float32), + bias=bias.to(torch.float32), + eps=eps, + ).view(-1, num_heads * linear_dim) + if concat_u and concat_x: + y = torch.cat([u, x, y], dim=1) + else: + mul_u = u + if mul_u_activation_type == "sigmoid": + mul_u = torch.sigmoid(u) + elif mul_u_activation_type == "silu": + mul_u = F.silu(u) + y = mul_u * F.layer_norm( + x, + normalized_shape=(x.shape[-1],), + weight=weight.to(torch.float32), + bias=bias.to(torch.float32), + eps=eps, + ) + if concat_u: + if silu_u: + u = F.silu(u) + if concat_x: + y = torch.cat([u, x, y], dim=1) + else: + y = torch.cat([u, y], dim=1) + elif concat_x: + y = torch.cat([x, y], dim=1) + y = F.dropout( + y, + p=dropout_ratio, + training=training, + ) + return y.to(dtype) + + +def pytorch_hstu_compute_output( + attn: torch.Tensor, + u: torch.Tensor, + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + output_weight: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, +) -> torch.Tensor: + dtype = x.dtype + y = pytorch_norm_mul_dropout( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=linear_dim, + ) + return torch.addmm(x, y, output_weight.to(x.dtype)).to(dtype) + + +def pytorch_swiglu( + x: torch.Tensor, + w_gate: torch.Tensor, + w_up: torch.Tensor, +) -> torch.Tensor: + gate = F.silu(F.linear(x, w_gate)) + up = F.linear(x, w_up) + return gate * up diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged.py new file mode 100644 index 000000000..82d82f402 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged.py @@ -0,0 +1,258 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Tuple + +import torch + + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +def pytorch_jagged_dense_bmm( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, +) -> torch.Tensor: + dtype = jagged.dtype + jagged = jagged.to(torch.float32) + dense = dense.to(torch.float32) + padded_jagged = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged, + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + bmm_out = torch.bmm(padded_jagged, dense) + jagged_bmm_out = torch.ops.fbgemm.dense_to_jagged( + bmm_out, [seq_offsets], total_L=jagged.shape[0] + )[0] + jagged_bmm_out = jagged_bmm_out.to(dtype) + return jagged_bmm_out + + +def pytorch_jagged_dense_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, +) -> torch.Tensor: + dtype = jagged.dtype + jagged = jagged.to(torch.float32) + dense = dense.to(torch.float32) + padded_jagged = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged, + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + out = padded_jagged + dense.unsqueeze(1) + jagged_out = torch.ops.fbgemm.dense_to_jagged( + out, [seq_offsets], total_L=jagged.shape[0] + )[0] + jagged_out = jagged_out.to(dtype) + return jagged_out + + +def pytorch_jagged_dense_bmm_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + elementwise: bool = False, +) -> torch.Tensor: + dtype = jagged.dtype + jagged = jagged.to(torch.float32) + dense = dense.to(torch.float32) + padded_jagged = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged, + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + bmm_out = torch.bmm(padded_jagged, dense) + + if elementwise: + jagged_out = ( + torch.ops.fbgemm.dense_to_jagged( + bmm_out, [seq_offsets], total_L=jagged.shape[0] + )[0] + + bias + ) + else: + jagged_out = torch.ops.fbgemm.dense_to_jagged( + bmm_out + bias.unsqueeze(1), [seq_offsets], total_L=jagged.shape[0] + )[0] + + jagged_out = jagged_out.to(dtype) + return jagged_out + + +@torch.fx.wrap +def _arange(len: int, device: torch.device) -> torch.Tensor: + return torch.arange(len, device=device) + + +def pytorch_concat_2D_dense_jagged( + jagged_max_seq_len: int, + jagged_offsets: torch.Tensor, + jagged_values: torch.Tensor, + dense_values: torch.Tensor, +) -> torch.Tensor: + B, dense_size, D = dense_values.size() + jagged_dense = torch.ops.fbgemm.jagged_to_padded_dense( + values=jagged_values, + offsets=[jagged_offsets], + max_lengths=[jagged_max_seq_len], + padding_value=0.0, + ) + concatted_dense = torch.cat([dense_values, jagged_dense], dim=1) + concatted_offsets = ( + dense_size * _arange(B + 1, device=jagged_offsets.device) + jagged_offsets + ) + return torch.ops.fbgemm.dense_to_jagged( + concatted_dense, + [concatted_offsets], + total_L=jagged_values.shape[0] + dense_size * B, + )[0] + + +def pytorch_concat_2D_jagged_jagged( + max_seq_len_left: int, + offsets_left: torch.Tensor, + values_left: torch.Tensor, + max_seq_len_right: int, + offsets_right: torch.Tensor, + values_right: torch.Tensor, + is_replace: bool = False, + n_prefix_from_right: int = 0, +) -> torch.Tensor: + # is_replace with n_prefix_from_right != 0 is not supported yet (neither in triton) + if is_replace: + return pytorch_replace_last_n_with_jagged( + max_seq_len_left, + offsets_left, + values_left, + offsets_right, + values_right, + ) + + lengths_a = offsets_left[1:] - offsets_left[:-1] + lengths_b = offsets_right[1:] - offsets_right[:-1] + + # Compute output offsets via cumsum (no dynamic shapes). + output_lengths = lengths_a + lengths_b + output_offsets = torch.nn.functional.pad( + torch.cumsum(output_lengths, dim=0), (1, 0) + ) + + total_len = values_left.shape[0] + values_right.shape[0] + positions = torch.arange(total_len, device=values_left.device) + batch_idx = torch.searchsorted(output_offsets[1:], positions, right=True) + local_pos = positions - output_offsets[batch_idx] + + per_batch_lengths_a = lengths_a[batch_idx] + + # Classify each output position into prefix / left / suffix. + is_prefix = local_pos < n_prefix_from_right + is_left = (local_pos >= n_prefix_from_right) & ( + local_pos < n_prefix_from_right + per_batch_lengths_a + ) + + # Pad with a sentinel zero row so index_select works on empty tensors + values_left_safe = torch.nn.functional.pad(values_left, (0, 0, 0, 1)) + values_right_safe = torch.nn.functional.pad(values_right, (0, 0, 0, 1)) + + left_idx = (offsets_left[batch_idx] + (local_pos - n_prefix_from_right)).clamp( + min=0, max=values_left.shape[0] + ) + right_prefix_idx = offsets_right[batch_idx] + local_pos + right_suffix_idx = offsets_right[batch_idx] + (local_pos - per_batch_lengths_a) + right_idx = torch.where(is_prefix, right_prefix_idx, right_suffix_idx).clamp( + min=0, max=values_right.shape[0] + ) + + left_values = values_left_safe.index_select(0, left_idx) + right_values = values_right_safe.index_select(0, right_idx) + + return torch.where(is_left.unsqueeze(-1), left_values, right_values) + + +def pytorch_jagged_remove_first_or_last_1D( + values: torch.Tensor, + lengths: torch.Tensor, + offsets: torch.Tensor, + max_seq_len: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + values = values.view(-1, 1) + shrunk_lengths = lengths - 1 + k_lengths = torch.stack([shrunk_lengths, torch.ones_like(lengths)], dim=1).view(-1) + q_lengths = torch.stack([torch.ones_like(lengths), shrunk_lengths], dim=1).view(-1) + all_indices = torch.arange( + start=0, end=q_lengths.numel(), device=values.device + ).reshape(-1, 2) + q_indices, k_indices = all_indices[:, 1], all_indices[:, 0] + values_no_first, _ = torch.ops.fbgemm.jagged_index_select( + values, q_lengths, q_indices + ) + values_no_last, _ = torch.ops.fbgemm.jagged_index_select( + values, k_lengths, k_indices + ) + return values_no_first.squeeze(), values_no_last.squeeze() + + +@torch.fx.wrap +def fx_apply_mask( + tensor: torch.Tensor, mask: torch.Tensor, fill_value: torch.Tensor +) -> torch.Tensor: + tensor[mask] = fill_value + return tensor + + +def pytorch_replace_last_n_with_jagged( + max_seq_len_left: int, + offsets_left: torch.Tensor, + values_left: torch.Tensor, + offsets_right: torch.Tensor, + values_right: torch.Tensor, +) -> torch.Tensor: + lengths_a = offsets_left[1:] - offsets_left[:-1] + lengths_b = offsets_right[1:] - offsets_right[:-1] + + total_len = values_left.shape[0] + positions = torch.arange(total_len, device=values_left.device) + batch_idx = torch.searchsorted(offsets_left[1:], positions, right=True) + local_pos = positions - offsets_left[batch_idx] + + # Positions >= (lengths_a - lengths_b) within each batch are in the replace zone. + threshold = lengths_a[batch_idx] - lengths_b[batch_idx] + in_replace_zone = local_pos >= threshold + + # Pad with a sentinel zero row so index_select works on empty tensors + values_right_safe = torch.nn.functional.pad(values_right, (0, 0, 0, 1)) + right_idx = (offsets_right[batch_idx] + (local_pos - threshold)).clamp( + min=0, max=values_right.shape[0] + ) + right_values = values_right_safe.index_select(0, right_idx) + return torch.where(in_replace_zone.unsqueeze(-1), right_values, values_left) diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged_tensors.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged_tensors.py new file mode 100644 index 000000000..27817f7fb --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_jagged_tensors.py @@ -0,0 +1,246 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +from generative_recommenders.common import fx_arange + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +def _concat_2D_jagged_jagged( + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: int, + max_len_right: int, + offsets_left: torch.Tensor, + offsets_right: torch.Tensor, +) -> torch.Tensor: + max_seq_len = max_len_left + max_len_right + lengths_left = offsets_left[1:] - offsets_left[:-1] + lengths_right = offsets_right[1:] - offsets_right[:-1] + padded_left = torch.ops.fbgemm.jagged_to_padded_dense( + values=values_left, + offsets=[offsets_left], + max_lengths=[max_len_left], + padding_value=0.0, + ) + padded_right = torch.ops.fbgemm.jagged_to_padded_dense( + values=values_right, + offsets=[offsets_right], + max_lengths=[max_len_right], + padding_value=0.0, + ) + concatted_dense = torch.cat([padded_left, padded_right], dim=1) + mask = fx_arange(max_seq_len, device=offsets_left.device).view(1, -1) + mask = torch.logical_or( + mask < lengths_left.view(-1, 1), + torch.logical_and( + mask >= max_len_left, + mask < max_len_left + lengths_right.view(-1, 1), + ), + ) + return concatted_dense.flatten(0, 1)[mask.view(-1), :] + + +@torch.fx.wrap +def pytorch_concat_2D_jagged( + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], +) -> torch.Tensor: + if offsets_left is None: + assert max_len_left is not None + B = values_left.shape[0] // max_len_left + offsets_left_non_optional = max_len_left * torch.arange( + B + 1, device=values_left.device + ) + else: + offsets_left_non_optional = offsets_left + if offsets_right is None: + assert max_len_right is not None + B = values_right.shape[0] // max_len_right + offsets_right_non_optional = max_len_right * torch.arange( + B + 1, device=values_left.device + ) + else: + offsets_right_non_optional = offsets_right + max_len_left = ( + int( + (offsets_left_non_optional[1:] - offsets_left_non_optional[:-1]) + .max() + .item() + ) + if max_len_left is None + else max_len_left + ) + max_len_right = ( + int( + (offsets_right_non_optional[1:] - offsets_right_non_optional[:-1]) + .max() + .item() + ) + if max_len_right is None + else max_len_right + ) + return _concat_2D_jagged_jagged( + values_left=values_left, + values_right=values_right, + max_len_left=max_len_left, + max_len_right=max_len_right, + offsets_left=offsets_left_non_optional, + offsets_right=offsets_right_non_optional, + ) + + +def _split_2D_jagged_jagged( + max_seq_len: int, + values: torch.Tensor, + offsets_left: torch.Tensor, + offsets_right: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor]: + offsets = offsets_left + offsets_right + padded_values = torch.ops.fbgemm.jagged_to_padded_dense( + values=values, + offsets=[offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ).flatten(0, 1) + lengths_left = offsets_left[1:] - offsets_left[:-1] + lengths_right = offsets_right[1:] - offsets_right[:-1] + mask = fx_arange(max_seq_len, device=values.device).view(1, -1) + mask_left = mask < lengths_left.view(-1, 1) + mask_right = torch.logical_and( + mask >= lengths_left.view(-1, 1), + mask < (lengths_left + lengths_right).view(-1, 1), + ) + return padded_values[mask_left.view(-1), :], padded_values[mask_right.view(-1), :] + + +@torch.fx.wrap +def pytorch_split_2D_jagged( + max_seq_len: int, + values: torch.Tensor, + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], +) -> Tuple[torch.Tensor, torch.Tensor]: + if offsets_left is None: + assert max_len_left is not None + assert offsets_right is not None + offsets_left_non_optional = max_len_left * torch.arange( + offsets_right.shape[0], device=values.device + ) + else: + offsets_left_non_optional = offsets_left + if offsets_right is None: + assert max_len_right is not None + assert offsets_left is not None + offsets_right_non_optional = max_len_right * torch.arange( + offsets_left.shape[0], device=values.device + ) + else: + offsets_right_non_optional = offsets_right + return _split_2D_jagged_jagged( + max_seq_len=max_seq_len, + values=values, + offsets_left=offsets_left_non_optional, + offsets_right=offsets_right_non_optional, + ) + + +def pytorch_hstu_split_l2_embeddings( + max_seq_len: int, + x: torch.Tensor, + prefix_offsets: torch.Tensor, + l2_offsets: torch.Tensor, + contextual_seq_len: int, +) -> Tuple[torch.Tensor, torch.Tensor]: + x_offsets = prefix_offsets + l2_offsets + x_lengths = x_offsets[1:] - x_offsets[:-1] + padded_x = torch.ops.fbgemm.jagged_to_padded_dense( + values=x, + offsets=[x_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ).flatten(0, 1) + prefix_lengths = prefix_offsets[1:] - prefix_offsets[:-1] + mask = fx_arange(max_seq_len, device=x_offsets.device).view(1, -1) + mask_prefix = torch.logical_and( + mask >= contextual_seq_len, + mask < prefix_lengths.view(-1, 1) + contextual_seq_len, + ) + mask_l2 = torch.logical_or( + mask < contextual_seq_len, + torch.logical_and( + mask >= prefix_lengths.view(-1, 1) + contextual_seq_len, + mask < x_lengths.view(-1, 1), + ), + ) + return padded_x[mask_prefix.view(-1), :], padded_x[mask_l2.view(-1), :] + + +def pytorch_hstu_concat_l2_embeddings( + max_prefix_len: int, + prefix_x: torch.Tensor, + prefix_offsets: torch.Tensor, + max_l2_len: int, + l2_x: torch.Tensor, + l2_offsets: torch.Tensor, + contextual_seq_len: int, +) -> torch.Tensor: + padded_prefix_x = torch.ops.fbgemm.jagged_to_padded_dense( + values=prefix_x, + offsets=[prefix_offsets], + max_lengths=[max_prefix_len], + padding_value=0.0, + ) + padded_l2_x = torch.ops.fbgemm.jagged_to_padded_dense( + values=l2_x, + offsets=[l2_offsets], + max_lengths=[max_l2_len], + padding_value=0.0, + ) + padded_x = torch.cat( + [ + padded_l2_x[:, 0:contextual_seq_len, :], + padded_prefix_x, + padded_l2_x[:, contextual_seq_len:, :], + ], + dim=1, + ) + mask = fx_arange(max_prefix_len + max_l2_len, device=prefix_x.device).view(1, -1) + prefix_lengths = prefix_offsets[1:] - prefix_offsets[:-1] + l2_lengths = l2_offsets[1:] - l2_offsets[:-1] + mask = torch.logical_or( + mask < prefix_lengths.view(-1, 1) + contextual_seq_len, + torch.logical_and( + mask >= max_prefix_len + contextual_seq_len, + mask < max_prefix_len + l2_lengths.view(-1, 1), + ), + ) + return padded_x.flatten(0, 1)[mask.view(-1), :] diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_layer_norm.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_layer_norm.py new file mode 100644 index 000000000..0666212ce --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_layer_norm.py @@ -0,0 +1,81 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +# pyre-strict + + +from typing import List + +import torch + + +def pytorch_layer_norm( + x: torch.Tensor, + normalized_shape: List[int], + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, +) -> torch.Tensor: + dtype = x.dtype + return torch.nn.functional.layer_norm( + x.to(torch.float32), + normalized_shape, + weight.to(torch.float32), + bias.to(torch.float32), + eps, + ).to(dtype) + + +def pytorch_rms_norm( + x: torch.Tensor, + normalized_shape: List[int], + weight: torch.Tensor, + eps: float, + silu: bool = False, +) -> torch.Tensor: + dtype = x.dtype + x_float = x.to(torch.float32) + normalized = torch.nn.functional.rms_norm( + x_float, + normalized_shape, + weight.to(torch.float32), + eps, + ) + if silu: + normalized = torch.nn.functional.silu(normalized) + return normalized.to(dtype) + + +def pytorch_swish_layer_norm( + x: torch.Tensor, + normalized_shape: List[int], + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, +) -> torch.Tensor: + dtype = x.dtype + x = x.to(torch.float32) + return ( + x + * torch.sigmoid( + torch.nn.functional.layer_norm( + x, + normalized_shape, + weight.to(torch.float32), + bias.to(torch.float32), + eps, + ) + ) + ).to(dtype) diff --git a/recommendation_v4/generative_recommenders/ops/pytorch/pt_position.py b/recommendation_v4/generative_recommenders/ops/pytorch/pt_position.py new file mode 100644 index 000000000..fced57e4f --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/pytorch/pt_position.py @@ -0,0 +1,142 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional + +import torch +from generative_recommenders.common import ( + fx_unwrap_optional_tensor, + jagged_to_padded_dense, +) + +try: + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops") + torch.ops.load_library("//deeplearning/fbgemm/fbgemm_gpu:sparse_ops_cpu") +except OSError: + pass + + +@torch.fx.wrap +def torch_arange(end: int, device: torch.device) -> torch.Tensor: + return torch.arange(end, device=device) + + +@torch.fx.wrap +def _get_col_indices( + max_seq_len: int, + max_contextual_seq_len: int, + max_pos_ind: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, +) -> torch.Tensor: + B = seq_lengths.size(0) + col_indices = torch.arange(max_seq_len, device=seq_lengths.device).expand( + B, max_seq_len + ) + if num_targets is not None: + if interleave_targets: + high_inds = seq_lengths - fx_unwrap_optional_tensor(num_targets) * 2 + else: + high_inds = seq_lengths - fx_unwrap_optional_tensor(num_targets) + col_indices = torch.clamp(col_indices, max=high_inds.view(-1, 1)) + col_indices = high_inds.view(-1, 1) - col_indices + else: + col_indices = seq_lengths.view(-1, 1) - col_indices + col_indices = col_indices + max_contextual_seq_len + col_indices = torch.clamp(col_indices, max=max_pos_ind - 1) + if max_contextual_seq_len > 0: + col_indices[:, :max_contextual_seq_len] = torch.arange( + 0, + max_contextual_seq_len, + device=col_indices.device, + dtype=col_indices.dtype, + ).view(1, -1) + return col_indices + + +def pytorch_add_timestamp_positional_embeddings( + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + pos_embeddings: torch.Tensor, + ts_embeddings: torch.Tensor, + timestamps: torch.Tensor, + max_seq_len: int, + max_contextual_seq_len: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str, +) -> torch.Tensor: + max_pos_ind = int(pos_embeddings.size(0)) + # position encoding + pos_inds = _get_col_indices( + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + max_pos_ind=max_pos_ind, + seq_lengths=seq_lengths, + num_targets=num_targets, + interleave_targets=interleave_targets, + ) + B, _ = pos_inds.shape + # timestamp encoding + num_time_buckets = ts_embeddings.size(1) - 1 + time_bucket_increments = 60.0 + time_bucket_divisor = 1.0 + time_delta = 0 + timestamps = jagged_to_padded_dense( + values=timestamps.unsqueeze(-1), + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ).squeeze(-1) + query_time = torch.gather( + timestamps, dim=1, index=(seq_lengths - 1).unsqueeze(1).clamp(min=0) + ) + ts = query_time - timestamps + ts = ts + time_delta + ts = ts.clamp(min=1e-6) / time_bucket_increments + if time_bucket_fn == "log": + ts = torch.log(ts) + else: + ts = torch.sqrt(ts) + ts = (ts / time_bucket_divisor).clamp(min=0).int() + ts = torch.clamp( + ts, + min=0, + max=num_time_buckets, + ) + position_embeddings = torch.index_select( + pos_embeddings, 0, pos_inds.reshape(-1) + ).view(B, max_seq_len, -1) + time_embeddings = torch.index_select(ts_embeddings, 0, ts.reshape(-1)).view( + B, max_seq_len, -1 + ) + padded_emb = torch.ops.fbgemm.jagged_to_padded_dense( + values=seq_embeddings, + offsets=[seq_offsets], + max_lengths=[max_seq_len], + padding_value=0.0, + ) + summed = padded_emb + (time_embeddings + position_embeddings).to( + seq_embeddings.dtype + ) + result, _ = torch.ops.fbgemm.dense_to_jagged( + summed, [seq_offsets], seq_embeddings.shape[0] + ) + return result diff --git a/recommendation_v4/generative_recommenders/ops/tests/fake_signature_test.py b/recommendation_v4/generative_recommenders/ops/tests/fake_signature_test.py new file mode 100644 index 000000000..fb9047454 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/fake_signature_test.py @@ -0,0 +1,162 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +""" +Tests to ensure fake and real implementations of triton functions +have the same function signatures. This is critical for PT2 compile compatibility. +""" + +import inspect +import unittest +from typing import Any, Callable, List + + +def get_custom_op_params(func: Callable[..., object]) -> List[str]: + """ + Get parameter names from a function, handling custom_op decorated functions. + + For maybe_register_custom_op decorated functions, inspect.signature may return + *args, **kwargs instead of the actual parameters. In this case, we need to + access the underlying schema to get the real parameter names. + """ + sig = inspect.signature(func) + params = list(sig.parameters.keys()) + + if params == ["args", "kwargs"]: + func_any: Any = func + if hasattr(func_any, "_opoverload"): + schema = func_any._opoverload._schema + return [arg.name for arg in schema.arguments] + + return params + + +class FakeSignatureTest(unittest.TestCase): + """Test to ensure fake and real implementations have the same function signatures.""" + + def test_triton_addmm_fwd_and_fake_have_same_signature(self) -> None: + """Verify triton_addmm_fwd and triton_addmm_fwd_fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_addmm import ( + triton_addmm_fwd, + triton_addmm_fwd_fake, + ) + + real_params = get_custom_op_params(triton_addmm_fwd) + fake_params = get_custom_op_params(triton_addmm_fwd_fake) + + self.assertEqual( + real_params, + fake_params, + f"triton_addmm_fwd and triton_addmm_fwd_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_maybe_triton_addmm_fwd_and_fake_have_same_signature(self) -> None: + """Verify maybe_triton_addmm_fwd and maybe_triton_addmm_fwd_fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_addmm import ( + maybe_triton_addmm_fwd, + maybe_triton_addmm_fwd_fake, + ) + + real_params = get_custom_op_params(maybe_triton_addmm_fwd) + fake_params = get_custom_op_params(maybe_triton_addmm_fwd_fake) + + self.assertEqual( + real_params, + fake_params, + f"maybe_triton_addmm_fwd and maybe_triton_addmm_fwd_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_triton_hstu_attention_fwd_and_fake_have_same_signature(self) -> None: + """Verify triton_hstu_attention_fwd and _triton_hstu_attention_fwd_fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_hstu_attention import ( + _triton_hstu_attention_fwd_fake, + triton_hstu_attention_fwd, + ) + + real_params = get_custom_op_params(triton_hstu_attention_fwd) + fake_params = get_custom_op_params(_triton_hstu_attention_fwd_fake) + + self.assertEqual( + real_params, + fake_params, + f"triton_hstu_attention_fwd and _triton_hstu_attention_fwd_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_triton_hstu_attention_bwd_and_fake_have_same_signature(self) -> None: + """Verify triton_hstu_attention_bwd and _triton_hstu_attention_bwd_fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_hstu_attention import ( + _triton_hstu_attention_bwd_fake, + triton_hstu_attention_bwd, + ) + + real_params = get_custom_op_params(triton_hstu_attention_bwd) + fake_params = get_custom_op_params(_triton_hstu_attention_bwd_fake) + + self.assertEqual( + real_params, + fake_params, + f"triton_hstu_attention_bwd and _triton_hstu_attention_bwd_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_triton_layer_norm_mul_dropout_fwd_impl_and_fake_have_same_signature( + self, + ) -> None: + """Verify _triton_layer_norm_mul_dropout_fwd_impl and its fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_hstu_linear import ( + _triton_layer_norm_mul_dropout_fwd_impl, + _triton_layer_norm_mul_dropout_fwd_impl_fake, + ) + + real_params = get_custom_op_params(_triton_layer_norm_mul_dropout_fwd_impl) + fake_params = get_custom_op_params(_triton_layer_norm_mul_dropout_fwd_impl_fake) + + self.assertEqual( + real_params, + fake_params, + f"_triton_layer_norm_mul_dropout_fwd_impl and _triton_layer_norm_mul_dropout_fwd_impl_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) + + def test_triton_layer_norm_mul_dropout_bwd_impl_and_fake_have_same_signature( + self, + ) -> None: + """Verify _triton_layer_norm_mul_dropout_bwd_impl and its fake have the same arguments.""" + from generative_recommenders.ops.triton.triton_hstu_linear import ( + _triton_layer_norm_mul_dropout_bwd_impl, + _triton_layer_norm_mul_dropout_bwd_impl_fake, + ) + + real_params = get_custom_op_params(_triton_layer_norm_mul_dropout_bwd_impl) + fake_params = get_custom_op_params(_triton_layer_norm_mul_dropout_bwd_impl_fake) + + self.assertEqual( + real_params, + fake_params, + f"_triton_layer_norm_mul_dropout_bwd_impl and _triton_layer_norm_mul_dropout_bwd_impl_fake have different arguments.\n" + f"Real: {real_params}\n" + f"Fake: {fake_params}", + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_test.py b/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_test.py new file mode 100644 index 000000000..ef70b8cc2 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_test.py @@ -0,0 +1,485 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import random +import unittest +from typing import Optional + +import torch +from generative_recommenders.common import ( + generate_sparse_seq_len, + gpu_unavailable, + HammerKernel, + set_dev_mode, +) +from generative_recommenders.ops.jagged_tensors import split_2D_jagged +from hypothesis import given, settings, strategies as st, Verbosity + + +def test_attn( + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + attn_dim: int, + hidden_dim: int, + causal: bool, + has_multiple_targets: bool, + has_max_attn_len: bool, + dtype: torch.dtype, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + sparsity: float = -1.0, + contextual_seq_len: int = 0, + atol: Optional[float] = None, + rtol: Optional[float] = None, + enable_tma: bool = False, +) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + from generative_recommenders.ops.hstu_attention import hstu_mha + + alpha = 1.0 / (attn_dim**0.5) + if sparsity > 0.0: + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=max_uih_len, + sparsity=sparsity, + device=torch.device("cuda"), + ) + else: + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + num_targets = torch.randint( + 1, max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + num_targets + contextual_seq_len + max_seq_len = max_uih_len + max_targets + contextual_seq_len + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + q = ( + torch.empty((L, heads, attn_dim), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + k = ( + torch.empty((L, heads, attn_dim), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + v = ( + torch.empty((L, heads, hidden_dim), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + + # ref implementation + ref_out = hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=causal, + num_targets=num_targets if has_multiple_targets else None, + dropout_pr=0.0, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=ref_kernel, + enable_tma=enable_tma, + ) + dout = torch.randn_like(ref_out) + ref_out.backward(dout) + + if skip_comparisons: + return + + # pyre-ignore + ref_dv, v.grad = v.grad.clone(), None + ref_dk, k.grad = k.grad.clone(), None + ref_dq, q.grad = q.grad.clone(), None + + # triton implementation + q = q.detach().clone().requires_grad_() + k = k.detach().clone().requires_grad_() + v = v.detach().clone().requires_grad_() + dout = dout.detach().clone() + real_out = hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=causal, + num_targets=num_targets if has_multiple_targets else None, + dropout_pr=0.0, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=real_kernel, + enable_tma=enable_tma, + ) + + torch.testing.assert_close( + ref_out, + real_out, + atol=atol, + rtol=rtol, + ) + if test_backward: + real_out.backward(dout) + real_dq, real_dk, real_dv = q.grad.clone(), k.grad.clone(), v.grad.clone() + torch.testing.assert_close(ref_dv, real_dv, atol=atol, rtol=rtol) + torch.testing.assert_close(ref_dk, real_dk, atol=atol, rtol=rtol) + torch.testing.assert_close(ref_dq, real_dq, atol=atol, rtol=rtol) + + +def test_delta_attn( + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + delta_size: int, + attn_dim: int, + hidden_dim: int, + has_multiple_targets: bool, + has_max_attn_len: bool, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + contextual_seq_len: int = 0, + atol: Optional[float] = None, + rtol: Optional[float] = None, + enable_tma: bool = False, +) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + from generative_recommenders.ops.hstu_attention import delta_hstu_mha + + alpha = 1.0 / (attn_dim**0.5) + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + num_targets = torch.randint( + 1, delta_size + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + delta_size + contextual_seq_len + max_seq_len = max_uih_len + delta_size + contextual_seq_len + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + delta_q = torch.empty( + (batch_size * delta_size, heads, attn_dim), + dtype=dtype, + device=torch.device("cuda"), + ).uniform_(-0.1, 0.1) + k = torch.empty( + (L, heads, attn_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + v = torch.empty( + (L, heads, hidden_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + + # ref implementation + ref_out = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if has_multiple_targets else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=ref_kernel, + enable_tma=enable_tma, + ) + + # real implementation + real_out = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if has_multiple_targets else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=real_kernel, + enable_tma=enable_tma, + ) + torch.testing.assert_close( + ref_out, + real_out, + atol=atol, + rtol=rtol, + ) + + +class HSTUAttentionTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([20, 100, 128, 256]), + max_targets=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + # pyre-ignore[2] + def test_attn_triton(self, *args, **kwargs) -> None: + test_attn( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.just(64), + heads=st.just(4), + max_uih_len=st.sampled_from([32768]), + max_targets=st.sampled_from([32]), + attn_dim=st.just(128), + hidden_dim=st.just(128), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from([torch.bfloat16]), + has_max_attn_len=st.sampled_from([True, False]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=5, + deadline=None, + ) + # pyre-ignore[2] + def test_attn_triton_long_seqs(self, *args, **kwargs) -> None: + test_attn( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + skip_comparisons=True, + sparsity=1.0, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([100, 128, 256]), + max_targets=st.sampled_from([20, 512]), + delta_size=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([False, True]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + # pyre-ignore[2] + def test_delta_attn_triton(self, *args, **kwargs) -> None: + test_delta_attn( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([20, 100, 128]), + max_targets=st.sampled_from([20, 512]), + delta_size=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64]), + hidden_dim=st.sampled_from([16, 32, 64]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([False, True]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + def test_cache( + self, + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + delta_size: int, + attn_dim: int, + hidden_dim: int, + has_multiple_targets: bool, + dtype: torch.dtype, + has_max_attn_len: bool, + contextual_seq_len: int, + ) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + from generative_recommenders.ops.hstu_attention import delta_hstu_mha, hstu_mha + + alpha = 1.0 / (attn_dim**0.5) + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + num_targets = torch.randint( + 1, delta_size + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + delta_size + contextual_seq_len + max_seq_len = max_uih_len + delta_size + contextual_seq_len + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + q = torch.empty( + (L, heads, attn_dim), + dtype=dtype, + device=torch.device("cuda"), + ).uniform_(-0.1, 0.1) + _, delta_q = split_2D_jagged( + max_seq_len=max_seq_len, + values=q.view(-1, heads * attn_dim), + max_len_left=None, + max_len_right=delta_size, + offsets_left=torch.ops.fbgemm.asynchronous_complete_cumsum( + lengths - delta_size + ), + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + delta_q = delta_q.view(-1, heads, attn_dim) + k = torch.empty( + (L, heads, attn_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + v = torch.empty( + (L, heads, hidden_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + prime_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + lengths - delta_size + ) + + # ref implementation + ref_out = hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=True, + num_targets=num_targets if has_multiple_targets else None, + dropout_pr=0.0, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.TRITON, + ) + _, delta_out = split_2D_jagged( + max_seq_len=max_seq_len, + values=ref_out.view(-1, heads * hidden_dim), + max_len_left=None, + max_len_right=delta_size, + offsets_left=prime_offsets, + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + delta_out = delta_out.view(-1, heads, hidden_dim) + + # real implementation + real_delta_out = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if has_multiple_targets else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + ) + torch.testing.assert_close( + delta_out, + real_delta_out, + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_tma_test.py b/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_tma_test.py new file mode 100644 index 000000000..8bb264af6 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/hstu_attention_tma_test.py @@ -0,0 +1,270 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import random +import unittest + +import torch +from generative_recommenders.common import ( + HammerKernel, + nv_gpu_unavailable, + set_dev_mode, +) +from generative_recommenders.ops.jagged_tensors import split_2D_jagged +from generative_recommenders.ops.tests.hstu_attention_test import ( + test_attn, + test_delta_attn, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class HSTUAttentionTmaTest(unittest.TestCase): + @unittest.skipIf(*nv_gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([20, 100, 128, 256]), + max_targets=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([True, False]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + # pyre-ignore[2] + def test_attn_triton_tma(self, *args, **kwargs) -> None: + test_attn( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + enable_tma=True, + ) + + @unittest.skipIf(*nv_gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.just(64), + heads=st.just(4), + max_uih_len=st.sampled_from([32768]), + max_targets=st.sampled_from([32]), + attn_dim=st.just(128), + hidden_dim=st.just(128), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from([torch.bfloat16]), + has_max_attn_len=st.sampled_from([True, False]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=5, + deadline=None, + ) + # pyre-ignore[2] + def test_attn_triton_long_seqs_tma(self, *args, **kwargs) -> None: + test_attn( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + skip_comparisons=True, + sparsity=1.0, + enable_tma=True, + ) + + @unittest.skipIf(*nv_gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([100, 128, 256]), + max_targets=st.sampled_from([20, 512]), + delta_size=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([False, True]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + # pyre-ignore[2] + def test_delta_attn_triton_tma(self, *args, **kwargs) -> None: + test_delta_attn( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + enable_tma=True, + ) + + @unittest.skipIf(*nv_gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([20, 100, 128]), + max_targets=st.sampled_from([20, 512]), + delta_size=st.sampled_from([20, 512]), + attn_dim=st.sampled_from([16, 32, 64]), + hidden_dim=st.sampled_from([16, 32, 64]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + has_max_attn_len=st.sampled_from([False, True]), + contextual_seq_len=st.sampled_from([0, 10]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=200, + deadline=None, + ) + def test_cache_tma( + self, + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + delta_size: int, + attn_dim: int, + hidden_dim: int, + has_multiple_targets: bool, + dtype: torch.dtype, + has_max_attn_len: bool, + contextual_seq_len: int, + ) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = True + torch.backends.cuda.matmul.allow_tf32 = True + from generative_recommenders.ops.hstu_attention import delta_hstu_mha, hstu_mha + + alpha = 1.0 / (attn_dim**0.5) + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + num_targets = torch.randint( + 1, delta_size + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + delta_size + contextual_seq_len + max_seq_len = max_uih_len + delta_size + contextual_seq_len + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + q = torch.empty( + (L, heads, attn_dim), + dtype=dtype, + device=torch.device("cuda"), + ).uniform_(-0.1, 0.1) + _, delta_q = split_2D_jagged( + max_seq_len=max_seq_len, + values=q.view(-1, heads * attn_dim), + max_len_left=None, + max_len_right=delta_size, + offsets_left=torch.ops.fbgemm.asynchronous_complete_cumsum( + lengths - delta_size + ), + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + delta_q = delta_q.view(-1, heads, attn_dim) + k = torch.empty( + (L, heads, attn_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + v = torch.empty( + (L, heads, hidden_dim), dtype=dtype, device=torch.device("cuda") + ).uniform_(-0.1, 0.1) + prime_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum( + lengths - delta_size + ) + + # ref implementation + ref_out = hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + causal=True, + num_targets=num_targets if has_multiple_targets else None, + dropout_pr=0.0, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.TRITON, + enable_tma=True, + ) + _, delta_out = split_2D_jagged( + max_seq_len=max_seq_len, + values=ref_out.view(-1, heads * hidden_dim), + max_len_left=None, + max_len_right=delta_size, + offsets_left=prime_offsets, + offsets_right=None, + kernel=HammerKernel.TRITON, + ) + delta_out = delta_out.view(-1, heads, hidden_dim) + + # real implementation + real_delta_out = delta_hstu_mha( + max_seq_len=max_seq_len, + alpha=alpha, + delta_q=delta_q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets if has_multiple_targets else None, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + enable_tma=True, + ) + torch.testing.assert_close( + delta_out, + real_delta_out, + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/hstu_compute_test.py b/recommendation_v4/generative_recommenders/ops/tests/hstu_compute_test.py new file mode 100644 index 000000000..57f217895 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/hstu_compute_test.py @@ -0,0 +1,503 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import random +import unittest +from typing import Optional + +import torch +from generative_recommenders.common import ( + generate_sparse_seq_len, + gpu_unavailable, + HammerKernel, + set_dev_mode, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class HSTUComputeTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=1000, max_value=1000), + D=st.integers(min_value=128, max_value=128), + L=st.integers(min_value=512, max_value=512), + concat_u=st.booleans(), + concat_x=st.booleans(), + mul_u_activation_type=st.sampled_from(["silu", "sigmoid", "none"]), + group_norm=st.booleans(), + num_heads=st.sampled_from([4]), + training=st.just(False), + recompute_y_in_backward=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=20, + ) + # pyre-ignore[2] + def test_compute_output(self, *args, **kwargs) -> None: + self._test_compute_output( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + opt_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.just(1500000), + D=st.just(512), + L=st.just(512), + concat_u=st.sampled_from([True]), + concat_x=st.sampled_from([True]), + mul_u_activation_type=st.sampled_from(["none"]), + group_norm=st.sampled_from([False]), + num_heads=st.sampled_from([4]), + training=st.just(False), + recompute_y_in_backward=st.sampled_from([False]), + dtype=st.just(torch.bfloat16), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=1, + ) + # pyre-ignore[2] + def test_long_sequences_compute_output(self, *args, **kwargs) -> None: + self._test_compute_output( + *args, + **kwargs, + test_backward=False, + ref_kernel=HammerKernel.TRITON, + opt_kernel=HammerKernel.TRITON, + skip_comparisons=True, + ) + + def _test_compute_output( + self, + N: int, + D: int, + L: int, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + group_norm: bool, + num_heads: int, + training: bool, + recompute_y_in_backward: bool, + dtype: torch.dtype, + test_backward: bool, + ref_kernel: HammerKernel, + opt_kernel: HammerKernel, + skip_comparisons: bool = False, + atol: Optional[float] = None, + rtol: Optional[float] = None, + ) -> None: + from generative_recommenders.ops.hstu_compute import hstu_compute_output + + torch.manual_seed(0) + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + dropout_ratio = 0.3 if training else 0.0 + attn = ( + torch.empty((N, L), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + u = ( + torch.empty((N, L), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_weight = ( + torch.empty( + (L if not group_norm else num_heads,), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_bias = ( + torch.empty( + (L if not group_norm else num_heads,), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_eps = 1e-6 + # When group_norm=True, only concat_ux = concat_u and concat_x is supported + if group_norm: + L_mult = 3 if (concat_u and concat_x) else 1 + else: + L_mult = 1 + if concat_u: + L_mult += 1 + if concat_x: + L_mult += 1 + output_weight = ( + torch.empty((L * L_mult, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + + # ref + ref_out = hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + dropout_ratio=dropout_ratio, + output_weight=output_weight, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=L // num_heads, + training=training, + recompute_y_in_backward=recompute_y_in_backward, + kernel=ref_kernel, + ) + dout = torch.randn_like(ref_out) * 0.1 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dattn, attn.grad = attn.grad.detach().clone(), None + ref_du, u.grad = u.grad.detach().clone(), None + ref_d_norm_w, norm_weight.grad = norm_weight.grad.detach().clone(), None + ref_d_norm_b, norm_bias.grad = norm_bias.grad.detach().clone(), None + ref_dx, x.grad = x.grad.detach().clone(), None + ref_d_output_w, output_weight.grad = output_weight.grad.detach().clone(), None + + # opt + attn = attn.detach().clone().requires_grad_() + u = u.detach().clone().requires_grad_() + norm_weight = norm_weight.detach().clone().requires_grad_() + norm_bias = norm_bias.detach().clone().requires_grad_() + output_weight = output_weight.detach().clone().requires_grad_() + x = x.detach().clone().requires_grad_() + opt_out = hstu_compute_output( + attn=attn, + u=u, + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + dropout_ratio=dropout_ratio, + output_weight=output_weight, + concat_u=concat_u, + concat_x=concat_x, + mul_u_activation_type=mul_u_activation_type, + group_norm=group_norm, + num_heads=num_heads, + linear_dim=L // num_heads, + training=training, + recompute_y_in_backward=recompute_y_in_backward, + kernel=opt_kernel, + ) + torch.testing.assert_close( + ref_out, + opt_out, + atol=atol, + rtol=rtol, + ) + + if test_backward: + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dattn, attn.grad = attn.grad.detach().clone(), None + opt_du, u.grad = u.grad.detach().clone(), None + opt_d_norm_w, norm_weight.grad = norm_weight.grad.detach().clone(), None + opt_d_norm_b, norm_bias.grad = norm_bias.grad.detach().clone(), None + opt_dx, x.grad = x.grad.detach().clone(), None + opt_d_output_w, output_weight.grad = ( + output_weight.grad.detach().clone(), + None, + ) + torch.testing.assert_close(ref_du, opt_du) + torch.testing.assert_close(ref_dattn, opt_dattn) + torch.testing.assert_close(ref_d_norm_w, opt_d_norm_w) + torch.testing.assert_close(ref_d_norm_b, opt_d_norm_b) + torch.testing.assert_close(ref_dx, opt_dx) + torch.testing.assert_close(ref_d_output_w, opt_d_output_w) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + heads=st.integers(1, 4), + max_uih_len=st.sampled_from([100, 128, 256, 1300]), + max_targets=st.sampled_from([20, 512]), + embedding_dim=st.sampled_from([16, 32, 64]), + attn_dim=st.sampled_from([16, 32, 64, 128]), + hidden_dim=st.sampled_from([16, 32, 64, 128]), + causal=st.sampled_from([True]), + has_multiple_targets=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + contextual_seq_len=st.sampled_from([0]), + has_max_attn_len=st.sampled_from([False, True]), + sort_by_length=st.sampled_from([True, False]), + recompute_uvqk_in_backward=st.sampled_from([True, False]), + recompute_normed_x_in_backward=st.sampled_from([True, False]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=150, + deadline=None, + ) + # pyre-ignore[2] + def test_preprocess_and_attention(self, *args, **kwargs) -> None: + self._test_hstu_preprocess_and_attention( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_hstu_preprocess_and_attention( + self, + batch_size: int, + heads: int, + max_uih_len: int, + max_targets: int, + embedding_dim: int, + attn_dim: int, + hidden_dim: int, + causal: bool, + has_multiple_targets: bool, + has_max_attn_len: bool, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + test_backward: bool, + contextual_seq_len: int, + sort_by_length: bool, + recompute_uvqk_in_backward: bool, + recompute_normed_x_in_backward: bool, + sparsity: float = -1.0, + atol: Optional[float] = None, + rtol: Optional[float] = None, + ) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + + from generative_recommenders.ops.hstu_compute import ( + hstu_preprocess_and_attention, + ) + + alpha = 1.0 / (attn_dim**0.5) + if sparsity > 0.0: + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=max_uih_len, + sparsity=sparsity, + device=torch.device("cuda"), + ) + else: + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + + num_targets = torch.randint( + max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + lengths = lengths + num_targets + max_seq_len = max_uih_len + max_targets + if has_max_attn_len: + max_attn_len = random.randint(1, max_uih_len // 5) + else: + max_attn_len = 0 + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + + L = int(seq_offsets[-1].item()) + + x = ( + torch.empty((L, embedding_dim), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_weight = ( + torch.empty((embedding_dim,), dtype=dtype, device=torch.device("cuda")) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_bias = ( + torch.empty( + (embedding_dim,), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + norm_eps = 1e-6 + uvqk_weight = ( + torch.empty( + ( + embedding_dim, + (hidden_dim * 2 + attn_dim * 2) * heads, + ), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + uvqk_bias = ( + torch.empty( + (hidden_dim * 2 + attn_dim * 2) * heads, + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + + # ref implementation + ref_u, ref_attn_output, _, _ = hstu_preprocess_and_attention( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + attn_alpha=alpha, + causal=causal, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + recompute_uvqk_in_backward=recompute_uvqk_in_backward, + recompute_normed_x_in_backward=recompute_normed_x_in_backward, + sort_by_length=sort_by_length, + kernel=ref_kernel, + ) + ref_out = ref_u + ref_attn_output + dout = torch.randn_like(ref_out) * 0.01 + ref_out.backward(dout) + + # pyre-ignore + ref_dx, x.grad = x.grad.clone(), None + ref_d_norm_weight, norm_weight.grad = norm_weight.grad.clone(), None + ref_d_norm_bias, norm_bias.grad = norm_bias.grad.clone(), None + ref_d_uvqk_weight, uvqk_weight.grad = uvqk_weight.grad.clone(), None + ref_d_uvqk_bias, uvqk_bias.grad = uvqk_bias.grad.clone(), None + + # real implementation + x = x.detach().clone().requires_grad_() + norm_weight = norm_weight.detach().clone().requires_grad_() + norm_bias = norm_bias.detach().clone().requires_grad_() + uvqk_weight = uvqk_weight.detach().clone().requires_grad_() + uvqk_bias = uvqk_bias.detach().clone().requires_grad_() + real_u, real_attn_output, _, _ = hstu_preprocess_and_attention( + x=x, + norm_weight=norm_weight, + norm_bias=norm_bias, + norm_eps=norm_eps, + num_heads=heads, + attn_dim=attn_dim, + hidden_dim=hidden_dim, + uvqk_weight=uvqk_weight, + uvqk_bias=uvqk_bias, + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + attn_alpha=alpha, + causal=causal, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + recompute_uvqk_in_backward=recompute_uvqk_in_backward, + recompute_normed_x_in_backward=recompute_normed_x_in_backward, + sort_by_length=sort_by_length, + kernel=real_kernel, + ) + real_out = real_u + real_attn_output + torch.testing.assert_close( + ref_u, + real_u, + atol=atol, + rtol=rtol, + ) + torch.testing.assert_close( + ref_attn_output, + real_attn_output, + atol=atol, + rtol=rtol, + ) + if test_backward: + # real implementation + dout = dout.detach().clone() + real_out.backward(dout) + ( + real_dx, + real_d_norm_weight, + real_d_norm_bias, + real_d_uvqk_weight, + real_d_uvqk_bias, + ) = ( + x.grad.clone(), + norm_weight.grad.clone(), + norm_bias.grad.clone(), + uvqk_weight.grad.clone(), + uvqk_bias.grad.clone(), + ) + torch.testing.assert_close(ref_dx, real_dx, atol=atol, rtol=rtol) + torch.testing.assert_close( + ref_d_norm_weight, real_d_norm_weight, atol=atol, rtol=rtol + ) + torch.testing.assert_close( + ref_d_norm_bias, real_d_norm_bias, atol=atol, rtol=rtol + ) + torch.testing.assert_close( + ref_d_uvqk_weight, real_d_uvqk_weight, atol=atol, rtol=rtol + ) + torch.testing.assert_close( + ref_d_uvqk_bias, real_d_uvqk_bias, atol=atol, rtol=rtol + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/jagged_tensors_test.py b/recommendation_v4/generative_recommenders/ops/tests/jagged_tensors_test.py new file mode 100644 index 000000000..e03d68d0b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/jagged_tensors_test.py @@ -0,0 +1,963 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest +from typing import Optional + +import torch +from generative_recommenders.common import ( + generate_sparse_seq_len, + gpu_unavailable, + HammerKernel, + set_dev_mode, +) +from generative_recommenders.ops.jagged_tensors import ( + concat_2D_jagged, + concat_2D_jagged_multirow, + split_2D_jagged, + split_2D_jagged_multirow, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class JaggedTensorsTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_len_a=st.integers(20, 100), + max_len_b=st.integers(20, 100), + D=st.integers(10, 30), + is_dense_a=st.sampled_from([True, False]), + is_dense_b=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_split_2D_jagged_triton(self, *args, **kwargs) -> None: + self._test_split_2D_jagged( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_split_2D_jagged( + self, + batch_size: int, + max_len_a: int, + max_len_b: int, + D: int, + is_dense_a: bool, + is_dense_b: bool, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + dtype: torch.dtype = torch.float32, + skip_comparisons: bool = False, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.jagged_tensors import split_2D_jagged + + max_seq_len = max_len_a + max_len_b + if not is_dense_a: + lengths_a = torch.randint( + 1, max_len_a + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_a = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_a[1:] = torch.cumsum(lengths_a, dim=0) + total_len_a = int(offsets_a[-1].item()) + else: + offsets_a = None + total_len_a = batch_size * max_len_a + is_dense_b = False + if not is_dense_b: + lengths_b = torch.randint( + 1, max_len_b + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_b = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_b[1:] = torch.cumsum(lengths_b, dim=0) + total_len_b = int(offsets_b[-1].item()) + else: + offsets_b = None + total_len_b = batch_size * max_len_b + values = ( + torch.empty( + (total_len_a + total_len_b, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + ref_values_a, ref_values_b = split_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values=values, + max_len_left=max_len_a if is_dense_a else None, + max_len_right=max_len_b if is_dense_b else None, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=ref_kernel, + ) + d_values_a = torch.randn_like(ref_values_a) + d_values_b = torch.randn_like(ref_values_b) + ref_values_a.backward(d_values_a, retain_graph=True) + ref_values_b.backward(d_values_b) + if skip_comparisons: + return + + assert values.grad is not None + ref_d_values, values.grad = values.grad.clone(), None + + values = values.detach().clone().requires_grad_() + real_values_a, real_values_b = split_2D_jagged( + max_seq_len=max_seq_len, + values=values, + max_len_left=max_len_a if is_dense_a else None, + max_len_right=max_len_b if is_dense_b else None, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=real_kernel, + ) + torch.testing.assert_close(ref_values_a, real_values_a) + torch.testing.assert_close(ref_values_b, real_values_b) + + if test_backward: + d_values_a = d_values_a.detach().clone() + d_values_b = d_values_b.detach().clone() + real_values_a.backward(d_values_a, retain_graph=True) + real_values_b.backward(d_values_b) + real_d_values = values.grad.clone() + torch.testing.assert_close(ref_d_values, real_d_values) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_len_a=st.integers(20, 100), + max_len_b=st.integers(20, 100), + D=st.integers(10, 30), + is_dense_a=st.sampled_from([True, False]), + is_dense_b=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_concat_2D_jagged_triton(self, *args, **kwargs) -> None: + self._test_concat_2D_jagged( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.sampled_from([130]), + max_len_a=st.sampled_from([32768]), + max_len_b=st.sampled_from([10]), + D=st.sampled_from([512]), + is_dense_a=st.sampled_from([True, False]), + is_dense_b=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_concat_2D_jagged_large_tensor(self, *args, **kwargs) -> None: + self._test_concat_2D_jagged( + *args, + **kwargs, + test_backward=True, + skip_comparisons=True, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + ) + + def _test_concat_2D_jagged( + self, + batch_size: int, + max_len_a: int, + max_len_b: int, + D: int, + is_dense_a: bool, + is_dense_b: bool, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + dtype: torch.dtype = torch.float32, + skip_comparisons: bool = False, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.jagged_tensors import concat_2D_jagged + + if not is_dense_a: + lengths_a = torch.randint( + 1, max_len_a + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_a = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_a[1:] = torch.cumsum(lengths_a, dim=0) + total_len_a = int(offsets_a[-1].item()) + else: + offsets_a = None + total_len_a = batch_size * max_len_a + values_a = ( + torch.empty( + (total_len_a, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if not is_dense_b: + lengths_b = torch.randint( + 1, max_len_b + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_b = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_b[1:] = torch.cumsum(lengths_b, dim=0) + total_len_b = int(offsets_b[-1].item()) + else: + offsets_b = None + total_len_b = batch_size * max_len_b + values_b = ( + torch.empty( + (total_len_b, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + ref_values = concat_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values_left=values_a, + values_right=values_b, + max_len_left=max_len_a, + max_len_right=max_len_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=ref_kernel, + ) + dout = torch.randn_like(ref_values) + ref_values.backward(dout) + if skip_comparisons: + return + + assert values_a.grad is not None + ref_d_a, values_a.grad = values_a.grad.clone(), None + assert values_b.grad is not None + ref_d_b, values_b.grad = values_b.grad.clone(), None + + values_a = values_a.detach().clone().requires_grad_() + values_b = values_b.detach().clone().requires_grad_() + dout = dout.detach().clone() + real_values = concat_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values_left=values_a, + values_right=values_b, + max_len_left=max_len_a, + max_len_right=max_len_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=real_kernel, + ) + torch.testing.assert_close(ref_values, real_values) + + if test_backward: + real_values.backward(dout) + real_d_a = values_a.grad.clone() + real_d_b = values_b.grad.clone() + torch.testing.assert_close(ref_d_a, real_d_a) + torch.testing.assert_close(ref_d_b, real_d_b) + + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_uih_len=st.integers(20, 100), + max_l2_len=st.integers(10, 30), + contextual_seq_len=st.sampled_from([0, 10]), + max_targets=st.sampled_from([10, 20]), + D=st.integers(10, 30), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + def test_hstu_split_l2_embeddings( + self, + batch_size: int, + max_uih_len: int, + max_l2_len: int, + contextual_seq_len: int, + max_targets: int, + D: int, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.jagged_tensors import hstu_split_l2_embeddings + + max_seq_len = max_uih_len + max_targets + contextual_seq_len + num_targets = torch.randint( + 1, max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + x_lengths = torch.randint( + 0, + max_uih_len + 1, + size=(batch_size,), + device=torch.device("cuda"), + ) + x_lengths = num_targets + x_lengths + contextual_seq_len + x_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + x_offsets[1:] = torch.cumsum(x_lengths, dim=0) + total_len = int(x_offsets[-1].item()) + x = ( + torch.empty( + (total_len, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + prefix_lengths = x_lengths - max_l2_len - num_targets - contextual_seq_len + prefix_lengths = torch.clamp(prefix_lengths, min=0) + prefix_offsets = torch.ops.fbgemm.asynchronous_complete_cumsum(prefix_lengths) + l2_offsets = x_offsets - prefix_offsets + ref_prefix_x, ref_l2_x = hstu_split_l2_embeddings( + max_seq_len=max_seq_len, + x=x, + prefix_offsets=prefix_offsets, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.PYTORCH, + ) + d_prefix_x = torch.randn_like(ref_prefix_x) + d_l2_x = torch.randn_like(ref_l2_x) + ref_prefix_x.backward(d_prefix_x, retain_graph=True) + ref_l2_x.backward(d_l2_x) + assert x.grad is not None + ref_d_x, x.grad = x.grad.clone(), None + x = x.detach().clone().requires_grad_() + real_prefix_x, real_l2_x = hstu_split_l2_embeddings( + max_seq_len=max_seq_len, + x=x, + prefix_offsets=prefix_offsets, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.TRITON, + ) + print(ref_prefix_x.shape, real_prefix_x.shape) + torch.testing.assert_close(ref_prefix_x, real_prefix_x) + torch.testing.assert_close(ref_l2_x, real_l2_x) + d_prefix_x = d_prefix_x.detach().clone() + d_l2_x = d_l2_x.detach().clone() + real_prefix_x.backward(d_prefix_x, retain_graph=True) + real_l2_x.backward(d_l2_x) + real_d_x = x.grad.clone() + torch.testing.assert_close(ref_d_x, real_d_x) + + # pyre-ignore + @given( + batch_size=st.integers(1, 1), + max_prefix_len=st.integers(10, 10), + max_l2_len=st.integers(5, 5), + contextual_seq_len=st.sampled_from([3]), + max_targets=st.sampled_from([2]), + D=st.integers(10, 10), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + def test_hstu_concat_l2_embeddings( + self, + batch_size: int, + max_prefix_len: int, + max_l2_len: int, + contextual_seq_len: int, + max_targets: int, + D: int, + dtype: torch.dtype, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.jagged_tensors import hstu_concat_l2_embeddings + + num_targets = torch.randint( + 1, max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + l2_lengths = torch.randint( + 0, + max_l2_len + 1, + size=(batch_size,), + device=torch.device("cuda"), + ) + l2_lengths = num_targets + l2_lengths + contextual_seq_len + max_l2_len = max_l2_len + contextual_seq_len + max_targets + l2_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + l2_offsets[1:] = torch.cumsum(l2_lengths, dim=0) + total_l2_len = int(l2_offsets[-1].item()) + l2_x = ( + torch.empty( + (total_l2_len, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + prefix_lengths = torch.randint( + 0, + max_prefix_len + 1, + size=(batch_size,), + device=torch.device("cuda"), + ) + prefix_lengths = torch.randint( + 0, + max_prefix_len + 1, + size=(batch_size,), + device=torch.device("cuda"), + ) + prefix_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + prefix_offsets[1:] = torch.cumsum(prefix_lengths, dim=0) + total_prefix_len = int(prefix_offsets[-1].item()) + prefix_x = ( + torch.empty( + (total_prefix_len, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + ref_x = hstu_concat_l2_embeddings( + max_prefix_len=max_prefix_len, + prefix_x=prefix_x, + prefix_offsets=prefix_offsets, + max_l2_len=max_l2_len, + l2_x=l2_x, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.PYTORCH, + ) + dout = torch.randn_like(ref_x) + ref_x.backward(dout) + + assert prefix_x.grad is not None + ref_d_prefix_x, prefix_x.grad = prefix_x.grad.clone(), None + assert l2_x.grad is not None + ref_d_l2_x, l2_x.grad = l2_x.grad.clone(), None + + prefix_x = prefix_x.detach().clone().requires_grad_() + l2_x = l2_x.detach().clone().requires_grad_() + real_x = hstu_concat_l2_embeddings( + max_prefix_len=max_prefix_len, + prefix_x=prefix_x, + prefix_offsets=prefix_offsets, + max_l2_len=max_l2_len, + l2_x=l2_x, + l2_offsets=l2_offsets, + contextual_seq_len=contextual_seq_len, + kernel=HammerKernel.TRITON, + ) + torch.testing.assert_close(ref_x, real_x) + dout = dout.detach().clone() + real_x.backward(dout) + real_d_prefix_x = prefix_x.grad.clone() + real_d_l2_x = l2_x.grad.clone() + torch.testing.assert_close(ref_d_prefix_x, real_d_prefix_x) + torch.testing.assert_close(ref_d_l2_x, real_d_l2_x) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(4, 8), + max_seq_len=st.integers(50, 500), + D=st.integers(20, 200), + K=st.integers(30, 200), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16, torch.float16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + contiguous=st.booleans(), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_jagged_dense_bmm_broadcast_add_triton(self, *args, **kwargs) -> None: + self._test_jagged_dense_bmm_broadcast_add( + *args, + **kwargs, + test_backward=True, + atol=None, + rtol=None, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.sampled_from([130]), + max_seq_len=st.sampled_from([32768]), + D=st.sampled_from([512]), + K=st.sampled_from([512]), + dtype=st.sampled_from([torch.float32, torch.bfloat16]), + contiguous=st.booleans(), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=1, + deadline=None, + ) + def test_jagged_dense_bmm_broadcast_add_triton_large_tensor( + self, + # pyre-fixme[2]: Parameter must be annotated. + *args, + **kwargs, # pyre-ignore[2] + ) -> None: + self._test_jagged_dense_bmm_broadcast_add( + *args, + **kwargs, + test_backward=True, + atol=None, + rtol=None, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + ) + + def _test_jagged_dense_bmm_broadcast_add( + self, + batch_size: int, + max_seq_len: int, + D: int, + K: int, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + test_backward: bool, + contiguous: bool = True, + atol: Optional[float] = None, + rtol: Optional[float] = None, + sparsity: float = -1, + ) -> None: + set_dev_mode(True) + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + from generative_recommenders.ops.jagged_tensors import ( + jagged_dense_bmm_broadcast_add, + ) + + if sparsity > 0.0: + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=max_seq_len, + sparsity=sparsity, + device=torch.device("cuda"), + ).to(torch.int64) + else: + lengths = torch.randint( + max_seq_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + # Test the edge case with an empty row + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + jagged_size = int(seq_offsets[-1].item()) + jagged = ( + torch.empty((jagged_size, D), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + dense = ( + torch.empty((batch_size, D, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + bias = ( + torch.empty((batch_size, K), dtype=dtype, device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if not contiguous: + dense = ( + dense.transpose(1, 2) + .contiguous() + .transpose(1, 2) + .detach() + .clone() + .requires_grad_() + ) + + ref_out = jagged_dense_bmm_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + kernel=ref_kernel, + ).to(jagged.dtype) + if test_backward: + dout = torch.randn_like(ref_out) * 0.01 + ref_out.backward(dout) + # pyre-ignore + ref_d_jagged, jagged.grad = jagged.grad.clone(), None + ref_d_dense, dense.grad = dense.grad.clone(), None + ref_d_bias, bias.grad = bias.grad.clone(), None + + jagged = jagged.detach().clone().requires_grad_() + dense = dense.detach().clone().requires_grad_() + bias = bias.detach().clone().requires_grad_() + real_out = jagged_dense_bmm_broadcast_add( + max_seq_len=max_seq_len, + seq_offsets=seq_offsets, + jagged=jagged, + dense=dense, + bias=bias, + kernel=real_kernel, + ) + torch.testing.assert_close( + ref_out, + real_out, + atol=atol, + rtol=rtol, + ) + if test_backward: + real_out.backward(dout) # pyre-ignore + real_d_jagged = jagged.grad.clone() + real_d_dense = dense.grad.clone() + real_d_bias = bias.grad.clone() + torch.testing.assert_close( + ref_d_jagged, # pyre-ignore + real_d_jagged, + atol=atol, + rtol=rtol, + ) + torch.testing.assert_close( + ref_d_dense, # pyre-ignore + real_d_dense, + atol=atol, + rtol=rtol, + ) + torch.testing.assert_close( + ref_d_bias, # pyre-ignore + real_d_bias, + atol=atol, + rtol=rtol, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_len_a=st.integers(20, 100), + max_len_b=st.integers(20, 100), + D=st.integers(10, 30), + is_dense_a=st.sampled_from([True, False]), + is_dense_b=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_concat_2D_jagged_multirow_triton(self, *args, **kwargs) -> None: + self._test_concat_2D_jagged_multirow( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_concat_2D_jagged_multirow( + self, + batch_size: int, + max_len_a: int, + max_len_b: int, + D: int, + is_dense_a: bool, + is_dense_b: bool, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + dtype: torch.dtype = torch.float32, + ) -> None: + set_dev_mode(True) + + if not is_dense_a: + lengths_a = torch.randint( + 1, max_len_a + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_a = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_a[1:] = torch.cumsum(lengths_a, dim=0) + total_len_a = int(offsets_a[-1].item()) + else: + offsets_a = None + total_len_a = batch_size * max_len_a + values_a = ( + torch.empty( + (total_len_a, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + if not is_dense_b: + lengths_b = torch.randint( + 1, max_len_b + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_b = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_b[1:] = torch.cumsum(lengths_b, dim=0) + total_len_b = int(offsets_b[-1].item()) + else: + offsets_b = None + total_len_b = batch_size * max_len_b + values_b = ( + torch.empty( + (total_len_b, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + ref_values = concat_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values_left=values_a, + values_right=values_b, + max_len_left=max_len_a, + max_len_right=max_len_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=ref_kernel, + ) + dout = torch.randn_like(ref_values) * 0.1 + ref_values.backward(dout) + assert values_a.grad is not None + ref_d_a, values_a.grad = values_a.grad.clone(), None + assert values_b.grad is not None + ref_d_b, values_b.grad = values_b.grad.clone(), None + + values_a = values_a.detach().clone().requires_grad_() + values_b = values_b.detach().clone().requires_grad_() + dout = dout.detach().clone() + + real_values = concat_2D_jagged_multirow( + max_seq_len=max_len_a + max_len_b, + values_left=values_a, + values_right=values_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + max_len_left=max_len_a, + max_len_right=max_len_b, + kernel=real_kernel, + ) + torch.testing.assert_close(ref_values, real_values) + if test_backward: + real_values.backward(dout) + real_d_a = values_a.grad.clone() + real_d_b = values_b.grad.clone() + torch.testing.assert_close(ref_d_a, real_d_a) + torch.testing.assert_close(ref_d_b, real_d_b) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + batch_size=st.integers(2, 8), + max_len_a=st.integers(20, 100), + max_len_b=st.integers(20, 100), + D=st.integers(10, 30), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_split_2D_jagged_multirow_triton(self, *args, **kwargs) -> None: + self._test_split_2D_jagged_multirow( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_split_2D_jagged_multirow( + self, + batch_size: int, + max_len_a: int, + max_len_b: int, + D: int, + test_backward: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + dtype: torch.dtype = torch.float32, + ) -> None: + set_dev_mode(True) + + lengths_a = torch.randint( + 1, max_len_a + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_a = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_a[1:] = torch.cumsum(lengths_a, dim=0) + + lengths_b = torch.randint( + 1, max_len_b + 1, size=(batch_size,), device=torch.device("cuda") + ) + offsets_b = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + offsets_b[1:] = torch.cumsum(lengths_b, dim=0) + + total_len = int(offsets_a[-1].item()) + int(offsets_b[-1].item()) + values = ( + torch.empty( + (total_len, D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + + ref_values_a, ref_values_b = split_2D_jagged( + max_seq_len=max_len_a + max_len_b, + values=values, + total_len_left=int(offsets_a[-1].item()), + total_len_right=int(offsets_b[-1].item()), + max_len_left=max_len_a, + max_len_right=max_len_b, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=ref_kernel, + ) + d_values_a = torch.randn_like(ref_values_a) * 0.1 + d_values_b = torch.randn_like(ref_values_b) * 0.1 + ref_values_a.backward(d_values_a, retain_graph=True) + ref_values_b.backward(d_values_b) + assert values.grad is not None + ref_d_values, values.grad = values.grad.clone(), None + + values = values.detach().clone().requires_grad_() + d_values_a = d_values_a.detach().clone() + d_values_b = d_values_b.detach().clone() + + max_len_a_actual = int((offsets_a[1:] - offsets_a[:-1]).max().item()) + max_len_b_actual = int((offsets_b[1:] - offsets_b[:-1]).max().item()) + + real_values_a, real_values_b = split_2D_jagged_multirow( + max_seq_len=max_len_a + max_len_b, + values=values, + total_len_left=int(offsets_a[-1].item()), + total_len_right=int(offsets_b[-1].item()), + max_len_left=max_len_a_actual, + max_len_right=max_len_b_actual, + offsets_left=offsets_a, + offsets_right=offsets_b, + kernel=real_kernel, + ) + torch.testing.assert_close(ref_values_a, real_values_a) + torch.testing.assert_close(ref_values_b, real_values_b) + if test_backward: + real_values_a.backward(d_values_a, retain_graph=True) + real_values_b.backward(d_values_b) + real_d_values = values.grad.clone() + torch.testing.assert_close(ref_d_values, real_d_values) diff --git a/recommendation_v4/generative_recommenders/ops/tests/layer_norm_test.py b/recommendation_v4/generative_recommenders/ops/tests/layer_norm_test.py new file mode 100644 index 000000000..62540967a --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/layer_norm_test.py @@ -0,0 +1,231 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import copy +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel, set_dev_mode +from generative_recommenders.ops.layer_norm import ( + layer_norm, + LayerNorm, + swish_layer_norm, + SwishLayerNorm, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class LayerNormTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.sampled_from([4200000]), + D=st.sampled_from([512]), + is_swish=st.sampled_from([False]), + dtype=st.sampled_from( + [torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=1, + ) + # pyre-ignore[2] + def test_large_tensors(self, *args, **kwargs) -> None: + self._test_layernorm( + *args, + **kwargs, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + skip_comparisons=True, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=0, max_value=10000), + D=st.integers(min_value=32, max_value=512), + is_swish=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=20, + ) + # pyre-ignore[2] + def test_ln(self, *args, **kwargs) -> None: + self._test_layernorm( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_layernorm( + self, + N: int, + D: int, + is_swish: bool, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + ) -> None: + N = N // 4 * 4 + # enable auto-tuning to verify correctness of multi-row kernel + set_dev_mode(False) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .normal_(0.0, 1.0) + .requires_grad_() + ) + weight = ( + torch.empty((D,), device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + bias = ( + torch.empty((D,), device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + if is_swish: + layer_norm_func = swish_layer_norm + else: + layer_norm_func = layer_norm + # ref + ref_out = layer_norm_func(x, weight, bias, eps=1e-6, kernel=ref_kernel) + dout = torch.randn_like(ref_out) * 0.05 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw, weight.grad = weight.grad.detach().clone(), None + ref_db, bias.grad = bias.grad.detach().clone(), None + # opt + x = x.detach().clone().requires_grad_() + weight = weight.detach().clone().requires_grad_() + bias = bias.detach().clone().requires_grad_() + opt_out = layer_norm_func(x, weight, bias, eps=1e-6, kernel=real_kernel) + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dx, x.grad = x.grad.detach().clone(), None + opt_dw, weight.grad = weight.grad.detach().clone(), None + opt_db, bias.grad = bias.grad.detach().clone(), None + torch.testing.assert_close(ref_out, opt_out) + torch.testing.assert_close(ref_dx, opt_dx) + torch.testing.assert_close(ref_dw, opt_dw) + torch.testing.assert_close(ref_db, opt_db) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=32, max_value=10000), + D=st.integers(min_value=32, max_value=512), + is_swish=st.sampled_from([True, False]), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=20, + ) + # pyre-ignore[2] + def test_modules(self, *args, **kwargs) -> None: + self._test_layer_norm_module( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_layer_norm_module( + self, + N: int, + D: int, + is_swish: bool, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + ) -> None: + set_dev_mode(True) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .normal_(0.0, 1.0) + .requires_grad_() + ) + # ref + if is_swish: + ref_layer = SwishLayerNorm( + dim=D, + eps=1e-6, + ).to(device="cuda") + ref_layer._hammer_kernel = ref_kernel + else: + ref_layer = LayerNorm( + dim=D, + eps=1e-6, + ).to(device="cuda") + ref_layer._hammer_kernel = ref_kernel + opt_layer = copy.deepcopy(ref_layer) + opt_layer._hammer_kernel = real_kernel + + ref_out = ref_layer(x) + dout = torch.randn_like(ref_out) * 0.05 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw = ref_layer.weight.grad.detach().clone() + ref_db = ref_layer.bias.grad.detach().clone() + # opt + x = x.detach().clone().requires_grad_() + opt_out = opt_layer(x) + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dx, x.grad = x.grad.detach().clone(), None + opt_dw = opt_layer.weight.grad.detach().clone() + opt_db = opt_layer.bias.grad.detach().clone() + torch.testing.assert_close(ref_out, opt_out) + torch.testing.assert_close( + ref_dx, + opt_dx, + ) + torch.testing.assert_close( + ref_dw, + opt_dw, + ) + torch.testing.assert_close( + ref_db, + opt_db, + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/mm_test.py b/recommendation_v4/generative_recommenders/ops/tests/mm_test.py new file mode 100644 index 000000000..0695275e3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/mm_test.py @@ -0,0 +1,156 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest +from typing import Optional + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel +from generative_recommenders.ops.mm import addmm +from hypothesis import given, settings, strategies as st, Verbosity + + +class MMlTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + M=st.integers(min_value=100, max_value=300), + N=st.integers(min_value=100, max_value=300), + K=st.sampled_from([128, 256]), + broadcast=st.booleans(), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16, torch.float16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + def test_addmm( + self, + M: int, + N: int, + K: int, + broadcast: bool, + dtype: torch.dtype, + ) -> None: + self._test_addmm( + M=M, + N=N, + K=K, + broadcast=broadcast, + dtype=dtype, + kernel_type=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + M=st.integers(min_value=100, max_value=300), + N=st.sampled_from([16, 48, 128, 144, 256]), + K=st.sampled_from([16, 48, 128, 144, 256]), + broadcast=st.booleans(), + dtype=st.sampled_from( + [torch.float32, torch.bfloat16, torch.float16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + def test_addmm_tma( + self, + M: int, + N: int, + K: int, + broadcast: bool, + dtype: torch.dtype, + ) -> None: + self._test_addmm( + M=M, + N=N, + K=K, + broadcast=broadcast, + dtype=dtype, + kernel_type=HammerKernel.TRITON, + ) + + def _test_addmm( + self, + M: int, + N: int, + K: int, + broadcast: bool, + dtype: torch.dtype, + kernel_type: HammerKernel, + atol: Optional[float] = None, + rtol: Optional[float] = None, + ) -> None: + # to enable more deterministic results. + torch.manual_seed(0) + + torch.backends.cudnn.allow_tf32 = False + torch.backends.cuda.matmul.allow_tf32 = False + + x: torch.Tensor = torch.rand((M, K), dtype=dtype, device="cuda").requires_grad_( + True + ) + w: torch.Tensor = torch.rand((K, N), dtype=dtype, device="cuda").requires_grad_( + True + ) + + if broadcast: + y: torch.Tensor = torch.rand( + (N), dtype=dtype, device="cuda" + ).requires_grad_(True) + else: + y: torch.Tensor = torch.rand( + (M, N), dtype=dtype, device="cuda" + ).requires_grad_(True) + + ref_z = addmm(y, x, w, kernel=HammerKernel.PYTORCH) + dz = torch.randn_like(ref_z) * 0.1 + ref_z.backward(dz) + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw, w.grad = w.grad.detach().clone(), None + ref_dy, y.grad = y.grad.detach().clone(), None + + x = x.detach().clone().requires_grad_(True) + w = w.detach().clone().requires_grad_(True) + y = y.detach().clone().requires_grad_(True) + real_z = addmm(y, x, w, kernel=kernel_type) + + torch.testing.assert_close(ref_z, real_z, atol=atol, rtol=rtol) + + # triton cc doesn't support backward + if kernel_type != HammerKernel.TRITON_CC: + real_z.backward(dz) + real_dx, x.grad = x.grad.detach().clone(), None + real_dw, w.grad = w.grad.detach().clone(), None + real_dy, y.grad = y.grad.detach().clone(), None + + torch.testing.assert_close(ref_dx, real_dx, atol=atol, rtol=rtol) + torch.testing.assert_close(ref_dw, real_dw, atol=atol, rtol=rtol) + torch.testing.assert_close(ref_dy, real_dy, atol=atol, rtol=rtol) diff --git a/recommendation_v4/generative_recommenders/ops/tests/position_test.py b/recommendation_v4/generative_recommenders/ops/tests/position_test.py new file mode 100644 index 000000000..ab9e1b415 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/position_test.py @@ -0,0 +1,234 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import ( + generate_sparse_seq_len, + gpu_unavailable, + HammerKernel, + set_dev_mode, +) +from hypothesis import given, settings, strategies as st, Verbosity + + +class PositionEmbeddingsTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + alpha=st.sampled_from([0.5]), + max_uih_len=st.integers(50, 500), + max_contextual_seq_len=st.sampled_from([10]), + interleave_targets=st.sampled_from([True, False]), + batch_size=st.integers(16, 32), + D=st.integers(20, 200), + max_targets=st.sampled_from([10, 20]), + time_bucket_fn=st.sampled_from(["log"]), + dtype=st.sampled_from([torch.float32, torch.bfloat16, torch.float16]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=20, + deadline=None, + ) + # pyre-ignore[2] + def test_add_timestamp_positional_embeddings_triton(self, *args, **kwargs) -> None: + self._test_add_timestamp_positional_embeddings( + *args, + **kwargs, + test_backward=True, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore + @given( + alpha=st.sampled_from([0.5]), + max_uih_len=st.sampled_from([32768]), + max_contextual_seq_len=st.sampled_from([10]), + interleave_targets=st.sampled_from([False]), + batch_size=st.sampled_from([130]), + D=st.sampled_from([512]), + max_targets=st.sampled_from([10]), + time_bucket_fn=st.sampled_from(["log"]), + dtype=st.sampled_from([torch.bfloat16]), + ) + @settings( + verbosity=Verbosity.verbose, + max_examples=1, + deadline=None, + ) + def test_add_timestamp_positional_embeddings_triton_large_tensor( + self, + # pyre-fixme[2]: Parameter must be annotated. + *args, + # pyre-ignore[2] + **kwargs, + ) -> None: + self._test_add_timestamp_positional_embeddings( + *args, + **kwargs, + test_backward=False, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + sparsity=1.0, + ) + + def _test_add_timestamp_positional_embeddings( + self, + alpha: float, + max_uih_len: int, + max_contextual_seq_len: int, + interleave_targets: bool, + batch_size: int, + D: int, + max_targets: int, + time_bucket_fn: str, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + test_backward: bool, + sparsity: float = -1, + ) -> None: + set_dev_mode(True) + from generative_recommenders.ops.position import ( + add_timestamp_positional_embeddings, + ) + + num_targets = torch.randint( + max_targets + 1, size=(batch_size,), device=torch.device("cuda") + ) + if sparsity > 0.0: + lengths = generate_sparse_seq_len( + size=batch_size, + max_seq_len=max_uih_len, + sparsity=sparsity, + device=torch.device("cuda"), + ).to(torch.int64) + else: + lengths = torch.randint( + max_uih_len + 1, size=(batch_size,), device=torch.device("cuda") + ) + seq_offsets = torch.zeros( + (batch_size + 1,), dtype=torch.int64, device=torch.device("cuda") + ) + seq_offsets[1:] = torch.cumsum(lengths, dim=0) + max_seq_len = max_uih_len + max_targets + + position_embeddings_weight = ( + torch.empty( + (max_seq_len, D), dtype=torch.float32, device=torch.device("cuda") + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + num_time_buckets = 1000 + timestamp_embeddings_weight = ( + torch.empty( + (num_time_buckets, D), dtype=torch.float32, device=torch.device("cuda") + ) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + seq_embeddings = ( + torch.empty( + (int(seq_offsets[-1].item()), D), + dtype=dtype, + device=torch.device("cuda"), + ) + .uniform_(-0.1, 0.1) + .requires_grad_() + ) + timestamp_deltas: torch.Tensor = torch.randint( + 86400, + size=(batch_size, max_seq_len), + device="cuda", + ) + timestamps = timestamp_deltas.cumsum(dim=1) + mask = torch.arange(max_seq_len, device=timestamps.device) < lengths.unsqueeze( + 1 + ) + timestamps = timestamps[mask.view(batch_size, -1)] + + ref_out = add_timestamp_positional_embeddings( + alpha=alpha, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + position_embeddings_weight=position_embeddings_weight, + timestamp_embeddings_weight=timestamp_embeddings_weight, + seq_offsets=seq_offsets, + seq_lengths=lengths, + seq_embeddings=seq_embeddings, + timestamps=timestamps, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + kernel=ref_kernel, + ) + dout = torch.randn_like(ref_out) * 0.01 + ref_out.backward(dout) + # pyre-ignore + ref_d_seq_embeddings, seq_embeddings.grad = seq_embeddings.grad.clone(), None + ref_d_position_embeddings_weight, position_embeddings_weight.grad = ( + position_embeddings_weight.grad.clone(), + None, + ) + ref_d_timestamp_embeddings_weight, timestamp_embeddings_weight.grad = ( + timestamp_embeddings_weight.grad.clone(), + None, + ) + + real_out = add_timestamp_positional_embeddings( + alpha=alpha, + max_seq_len=max_seq_len, + max_contextual_seq_len=max_contextual_seq_len, + position_embeddings_weight=position_embeddings_weight, + timestamp_embeddings_weight=timestamp_embeddings_weight, + seq_offsets=seq_offsets, + seq_lengths=lengths, + seq_embeddings=seq_embeddings, + timestamps=timestamps, + num_targets=num_targets, + interleave_targets=interleave_targets, + time_bucket_fn=time_bucket_fn, + kernel=real_kernel, + ) + + torch.testing.assert_close(ref_out, real_out) + if test_backward: + real_out.backward(dout) + real_d_seq_embeddings = seq_embeddings.grad.clone() + real_d_position_embeddings_weight = position_embeddings_weight.grad.clone() + real_d_timestamp_embeddings_weight = ( + timestamp_embeddings_weight.grad.clone() + ) + torch.testing.assert_close(ref_d_seq_embeddings, real_d_seq_embeddings) + torch.testing.assert_close( + ref_d_position_embeddings_weight, + real_d_position_embeddings_weight, + atol=5e-2 if dtype != torch.float32 else None, + rtol=2e-2 if dtype != torch.float32 else None, + ) + torch.testing.assert_close( + ref_d_timestamp_embeddings_weight, + real_d_timestamp_embeddings_weight, + atol=5e-2 if dtype != torch.float32 else None, + rtol=2e-2 if dtype != torch.float32 else None, + ) diff --git a/recommendation_v4/generative_recommenders/ops/tests/rms_norm_test.py b/recommendation_v4/generative_recommenders/ops/tests/rms_norm_test.py new file mode 100644 index 000000000..4e5c1a871 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/tests/rms_norm_test.py @@ -0,0 +1,229 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import copy +import unittest + +import torch +from generative_recommenders.common import gpu_unavailable, HammerKernel, set_dev_mode +from generative_recommenders.ops.layer_norm import rms_norm, RMSNorm +from hammer.ops.triton.cc.utils import set_triton_cc_version +from hypothesis import given, settings, strategies as st, Verbosity + + +class LayerNormTest(unittest.TestCase): + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.sampled_from([2000000]), + D=st.sampled_from([512]), + dtype=st.sampled_from( + [torch.bfloat16] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + silu=st.booleans(), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=1, + ) + # pyre-ignore[2] + def test_large_tensors(self, *args, **kwargs) -> None: + self._test_rms_norm( + *args, + **kwargs, + ref_kernel=HammerKernel.TRITON, + real_kernel=HammerKernel.TRITON, + skip_comparisons=True, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=0, max_value=10000), + D=st.integers(min_value=32, max_value=512), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + silu=st.booleans(), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=50, + ) + # pyre-ignore[2] + def test_rms_norm(self, *args, **kwargs) -> None: + self._test_rms_norm( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=4, max_value=10000), + D=st.sampled_from([256, 512]), + dtype=st.sampled_from([torch.bfloat16, torch.float16]), + triton_cc_version=st.sampled_from(["", "repkg"]), + silu=st.just(False), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=10, + ) + # pyre-ignore[2] + def test_rms_norm_triton_cc(self, triton_cc_version: str, *args, **kwargs) -> None: + set_triton_cc_version(triton_cc_version) + self._test_rms_norm( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON_CC, + test_backward=False, + ) + + def _test_rms_norm( + self, + N: int, + D: int, + dtype: torch.dtype, + silu: bool, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + test_backward: bool = True, + ) -> None: + N = N // 4 * 4 + # enable auto-tuning to verify correctness of multi-row kernel + set_dev_mode(False) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .normal_(0.0, 1.0) + .requires_grad_() + ) + weight = ( + torch.empty((D,), device=torch.device("cuda")) + .uniform_(-1.0, 1.0) + .requires_grad_() + ) + # ref + ref_out = rms_norm(x, weight, eps=1e-6, silu=silu, kernel=ref_kernel) + opt_x = x.detach().clone().requires_grad_() + opt_weight = weight.detach().clone().requires_grad_() + opt_out = rms_norm(opt_x, opt_weight, eps=1e-6, silu=silu, kernel=real_kernel) + torch.testing.assert_close(ref_out, opt_out) + + if not test_backward: + return + + dout = torch.randn_like(ref_out) * 0.05 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw, weight.grad = weight.grad.detach().clone(), None + # opt + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dx, x.grad = opt_x.grad.detach().clone(), None + opt_dw, weight.grad = opt_weight.grad.detach().clone(), None + torch.testing.assert_close(ref_dx, opt_dx) + torch.testing.assert_close(ref_dw, opt_dw) + + @unittest.skipIf(*gpu_unavailable) + # pyre-ignore[56] + @given( + N=st.integers(min_value=32, max_value=10000), + D=st.integers(min_value=32, max_value=512), + dtype=st.sampled_from( + [torch.bfloat16, torch.float32] + if torch.cuda.get_device_capability(torch.device("cuda"))[0] >= 8 + else [torch.float32] + ), + ) + @settings( + deadline=None, + verbosity=Verbosity.verbose, + max_examples=50, + ) + # pyre-ignore[2] + def test_modules(self, *args, **kwargs) -> None: + self._test_rms_norm_module( + *args, + **kwargs, + ref_kernel=HammerKernel.PYTORCH, + real_kernel=HammerKernel.TRITON, + ) + + def _test_rms_norm_module( + self, + N: int, + D: int, + dtype: torch.dtype, + ref_kernel: HammerKernel, + real_kernel: HammerKernel, + skip_comparisons: bool = False, + ) -> None: + set_dev_mode(True) + x = ( + torch.empty((N, D), dtype=dtype, device=torch.device("cuda")) + .normal_(0.0, 1.0) + .requires_grad_() + ) + # ref + ref_layer = RMSNorm( + dim=D, + eps=1e-6, + ).to(device="cuda") + ref_layer._hammer_kernel = ref_kernel + opt_layer = copy.deepcopy(ref_layer) + opt_layer._hammer_kernel = real_kernel + + ref_out = ref_layer(x) + dout = torch.randn_like(ref_out) * 0.05 + ref_out.backward(dout) + if skip_comparisons: + return + # pyre-ignore[16] + ref_dx, x.grad = x.grad.detach().clone(), None + ref_dw = ref_layer.weight.grad.detach().clone() + # opt + x = x.detach().clone().requires_grad_() + opt_out = opt_layer(x) + dout = dout.detach().clone() + opt_out.backward(dout) + opt_dx, x.grad = x.grad.detach().clone(), None + opt_dw = opt_layer.weight.grad.detach().clone() + torch.testing.assert_close(ref_out.to(dtype), opt_out) + torch.testing.assert_close( + ref_dx, + opt_dx, + ) + torch.testing.assert_close( + ref_dw, + opt_dw, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/_autotune_pinning.py b/recommendation_v4/generative_recommenders/ops/triton/_autotune_pinning.py new file mode 100644 index 000000000..5aa24eb85 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/_autotune_pinning.py @@ -0,0 +1,27 @@ +"""Triton autotune pinning helper. + +A handful of Triton kernels in this directory have two stable autotune +equilibria on MI350X gfx950 at our yambda bs=32 L=2039 shape: a fast one +(~52 ms/step) and a slow one (~71 ms/step). The autotuner's measurement +noise puts the choice on a coin flip per cold start. We pin the winning +config for these kernels so every cold start lands at the fast equilibrium +deterministically. + +Set `TRITON_FULL_AUTOTUNE=1` to bypass the pin and re-enable the full +autotune search (useful when validating a new shape, GPU, or Triton version +before re-capturing winners). +""" + +import os +from typing import Callable, List + +import triton + + +def pinned_or_full( + pinned: List[triton.Config], + full_configs_fn: Callable[[], List[triton.Config]], +) -> List[triton.Config]: + if os.environ.get("TRITON_FULL_AUTOTUNE", "0") == "1": + return full_configs_fn() + return pinned diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py b/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py new file mode 100644 index 000000000..487aae189 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_addmm.py @@ -0,0 +1,1708 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +import math +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.ops.utils import is_sm100_plus, maybe_register_custom_op + +try: + # @manual=//triton:triton + from triton.language.extra.subtile_ops import _split_n_2D +except ImportError: + _split_n_2D = None + +try: + # @manual=//triton:triton + import triton.language.extra.tlx as tlx # type: ignore + + HAS_TLX = True +except ImportError: + tlx = None + HAS_TLX = False + +from generative_recommenders.common import triton_autotune, triton_cc + +try: + # @manual=//triton:triton + from triton.tools.tensor_descriptor import TensorDescriptor + + TMA_AVAILABLE = True +except ImportError: + TMA_AVAILABLE = False + pass + + +ENABLE_FULL_TURNING_SPACE = False + + +def _use_meta_ws() -> bool: + """Check if Meta's warp specialization is available, enabled, and on SM100+.""" + return ( + is_sm100_plus() + and hasattr(triton, "knobs") + and hasattr(triton.knobs, "nvidia") + # `use_meta_ws` is absent in some Triton builds (e.g. nvcr.io/nvidia/pytorch:26.01-py3); + # use getattr so import doesn't crash on AttributeError before any step runs. + and getattr(triton.knobs.nvidia, "use_meta_ws", False) + ) + + +def _check_tma_alignment( + x: torch.Tensor, w: torch.Tensor, y: torch.Tensor, min_alignment: int = 16 +) -> bool: + """Check if tensors meet TMA alignment requirements. + + TMA (Tensor Memory Accelerator) on H100 requires: + 1. Base addresses to be 64-byte aligned + 2. Dimensions to be multiples of 64 for optimal performance + 3. Contiguous inner dimensions (stride=1) + + Args: + x: Input tensor [M, K] + w: Weight tensor [K, N] + y: Bias tensor [N] or [M, N] + min_alignment: Minimum alignment requirement (default: 64) + + Returns: + True if all tensors meet TMA alignment requirements + """ + _, K = x.shape + KB, N = w.shape + assert K == KB, f"incompatible dimensions {K}, {KB}" + + is_y_1d = y.dim() == 1 + NY = y.shape[0] if is_y_1d else y.shape[1] + assert N == NY, f"incompatible dimensions {N}, {NY}" + + return (K % min_alignment == 0) and (N % min_alignment == 0) + + +def _prune_persistent_autows_configs(configs, named_args, **kwargs): # noqa + if not _use_meta_ws(): + return configs + BROADCAST_Y = kwargs.get("BROADCAST_Y", False) + pruned = [] + for c in configs: + BLOCK_M = c.kwargs.get("BLOCK_M", 0) + BLOCK_N = c.kwargs.get("BLOCK_N", 0) + EPILOGUE_SUBTILE = c.kwargs.get("EPILOGUE_SUBTILE", 1) + DP = c.kwargs.get("DATA_PARTITION_FACTOR", 1) + # DATA_PARTITION_FACTOR=2 is only supported with BLOCK_M=256 + if DP == 2 and BLOCK_M != 256: + continue + if (BLOCK_N // EPILOGUE_SUBTILE) < 32: + continue + if BROADCAST_Y and (BLOCK_N // EPILOGUE_SUBTILE) < 64: + continue + pruned.append(c) + return pruned + + +def _prune_configs_for_tlx_persistent_addmm(configs, named_args, **kwargs): # noqa + M = named_args.get("M", 0) + N = named_args.get("N", 0) + BROADCAST_Y = kwargs.get("BROADCAST_Y", False) + + pruned = [] + for c in configs: + BLOCK_M = c.kwargs.get("BLOCK_M", 0) + BLOCK_N = c.kwargs.get("BLOCK_N", 0) + EPILOGUE_SUBTILE = c.kwargs.get("EPILOGUE_SUBTILE", 1) + NUM_MMA_GROUPS = c.kwargs.get("NUM_MMA_GROUPS", 1) + BLOCK_M_SPLIT = BLOCK_M // NUM_MMA_GROUPS + NUM_SMEM_BUFFERS = c.kwargs.get("NUM_SMEM_BUFFERS", 1) + + # Hardware constraint: Always make MMA tile 128. + if BLOCK_M_SPLIT != 128: + continue + + # BLOCK_N >= 64 required for PAIR_CTA + if BLOCK_N < 64: + continue + + # Subslice loads cannot be smaller than 32 + if (BLOCK_N // EPILOGUE_SUBTILE) < 32: + continue + + # TMA loads must be at least 128 bytes. With BROADCAST_Y + # this may not be met. + if BROADCAST_Y and (BLOCK_N // EPILOGUE_SUBTILE) < 64: + continue + + # Prune the support SMEM_BUFFER configurations. + if BROADCAST_Y: + if NUM_MMA_GROUPS == 1 and NUM_SMEM_BUFFERS != 5: + continue + elif NUM_MMA_GROUPS == 2 and NUM_SMEM_BUFFERS != 4: + continue + else: + if NUM_MMA_GROUPS == 1 and NUM_SMEM_BUFFERS != 4: + continue + elif NUM_MMA_GROUPS == 2 and NUM_SMEM_BUFFERS != 3: + continue + + # PAIR_CTA requires even number of M tiles and even total tiles + num_tiles_m = math.ceil(M / BLOCK_M) if BLOCK_M > 0 else 0 + num_tiles_n = math.ceil(N / BLOCK_N) if BLOCK_N > 0 else 0 + total_tiles = num_tiles_m * num_tiles_n + + # PAIR_CTA incompatible with MMA M=64 + pair_cta_compatible = ( + (num_tiles_m % 2 == 0) + and (total_tiles % 2 == 0) + and BLOCK_M == 128 + and NUM_MMA_GROUPS == 1 + ) + + c.kwargs["PAIR_CTA"] = pair_cta_compatible + # Set ctas_per_cga for CUDA-native cluster launch semantics (TLX way) + c.ctas_per_cga = (2, 1, 1) if pair_cta_compatible else None + + pruned.append(c) + return pruned + + +def get_mm_configs(pre_hook=None) -> List[triton.Config]: + if torch.version.hip: + if ENABLE_FULL_TURNING_SPACE: + block_m_range = [32, 64, 128, 256] + block_n_range = [32, 64, 128, 256] + block_k_range = [32, 64] + group_m_range = [4, 8] + matrix_instr_nonkdim_range = [16] + waves_per_eu_range = [0] + kpack_range = [1, 2] + num_warps_range = [4, 8] + num_stage_range = [2] if triton.__version__ >= "3.2.0" else [0] + else: + block_m_range = [256] + block_n_range = [256] + block_k_range = [32] + group_m_range = [8] + matrix_instr_nonkdim_range = [16] + waves_per_eu_range = [0] + kpack_range = [2] + num_warps_range = [8] + num_stage_range = [2] if triton.__version__ >= "3.2.0" else [0] + + return [ + triton.Config( + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": group_m, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "waves_per_eu": waves_per_eu, + "kpack": kpack, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=pre_hook, + ) + for block_m in block_m_range + for block_n in block_n_range + for block_k in block_k_range + for group_m in group_m_range + for matrix_instr_nonkdim in matrix_instr_nonkdim_range + for waves_per_eu in waves_per_eu_range + for kpack in kpack_range + for num_stages in num_stage_range + for num_warps in num_warps_range + ] + else: + block_m_range = [32, 64, 128, 256] + block_n_range = [32, 64, 128, 256] + block_k_range = [32, 64] + group_m_range = [4, 8] + # WARP_SPECIALIZE only works with num_warps >=4 + num_warps_range = [4, 8] if is_sm100_plus() else [2, 4, 8] + num_stage_range = [2, 3, 4, 5] + if ENABLE_FULL_TURNING_SPACE: + return [ + triton.Config( + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": group_m, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=pre_hook, + ) + for block_m in block_m_range + for block_n in block_n_range + for block_k in block_k_range + for group_m in group_m_range + for num_stages in num_stage_range + for num_warps in num_warps_range + ] + else: + configs = [ + triton.Config( + { + "BLOCK_M": 32, + "BLOCK_N": 64, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=5, + num_warps=2, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 256, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=3, + num_warps=8, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 256, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 128, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 128, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 32, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=4, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 32, + "BLOCK_K": 32, + "GROUP_M": 8, + }, + num_stages=5, + num_warps=2, + pre_hook=pre_hook, + ), + ] + if is_sm100_plus(): + configs += [ + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 256, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=3, + num_warps=4, + pre_hook=pre_hook, + ), + ] + return [c for c in configs if c.num_warps >= 4] + + return configs + + +def _get_addmm_tma_ws_persistent_configs(pre_hook=None) -> List[triton.Config]: + """Get configs for _addmm_fwd_tma_ws_persistent (sm100+ TLX kernel). + + This kernel has unique requirements (warp specialization, PAIR_CTA, + EPILOGUE_SUBTILE) that don't apply to the other addmm kernels. + """ + if ENABLE_FULL_TURNING_SPACE: + block_m_range = [64, 128, 256] + block_n_range = [64, 128, 256] + block_k_range = [64, 128, 256] + group_m_range = [8] + num_warps_range = [4] + num_stage_range = [1] + epilogue_subtile_range = [1, 2, 4] + num_mma_groups_range = [1, 2] + return [ + triton.Config( + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": group_m, + "EPILOGUE_SUBTILE": epilogue_subtile, + "NUM_MMA_GROUPS": num_mma_groups, + "NUM_TMEM_BUFFERS": 1 if num_mma_groups == 2 else 2, + "NUM_SMEM_BUFFERS": num_smem_buffers, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=pre_hook, + ) + for block_m in block_m_range + for block_n in block_n_range + for block_k in block_k_range + for group_m in group_m_range + for num_stages in num_stage_range + for num_warps in num_warps_range + for epilogue_subtile in epilogue_subtile_range + for num_mma_groups in num_mma_groups_range + for num_smem_buffers in [3, 4, 5] + ] + else: + configs = [] + for block_m, block_n, block_k in [ + (128, 256, 64), + (128, 128, 64), + (64, 128, 64), + (64, 256, 64), + (128, 64, 128), + ]: + # Note: num_smem_buffers is pruned to 1 in + # the pruning function. + for num_smem_buffers in [3, 4, 5]: + configs.append( + triton.Config( + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": 8, + "EPILOGUE_SUBTILE": 1, + "NUM_MMA_GROUPS": 1, + "NUM_TMEM_BUFFERS": 2, + "NUM_SMEM_BUFFERS": num_smem_buffers, + }, + num_stages=1, + num_warps=4, + pre_hook=pre_hook, + ), + ) + return configs + + +def get_triton_persistent_configs(pre_hook=None) -> List[triton.Config]: + if not _use_meta_ws(): + configs = get_mm_configs(pre_hook=pre_hook) + for c in configs: + c.kwargs["DATA_PARTITION_FACTOR"] = 1 + c.kwargs["EPILOGUE_SUBTILE"] = 1 + return configs + # TODO: Prune configs to best configs. + return [ + triton.Config( # pyre-ignore[28] + { + "BLOCK_M": block_m, + "BLOCK_N": block_n, + "BLOCK_K": block_k, + "GROUP_M": 8, + "EPILOGUE_SUBTILE": subtile, + "DATA_PARTITION_FACTOR": DP, + }, + num_stages=num_stages, + num_warps=4, + pre_hook=pre_hook, + early_tma_store_lowering=1, + maxRegAutoWS=255, + ) + for block_m in [64, 128, 256] + for block_n in [64, 128, 256] + for block_k in [64, 128, 256] + for num_stages in [2, 3, 4] + for subtile in [1, 2, 4, 8] + for DP in [1, 2] + ] + + +@triton_cc( + annotations={ + "M": "i32", + "N": ("i32", 16), + "K": ("i32", 16), + "stride_xm": ("i32", 16), + "stride_xk": ("i32", 1), + "stride_wk": ("i32", 16), + "stride_wn": ("i32", 1), + "stride_ym": ("i32", 16), + "stride_yn": ("i32", 1), + "stride_zm": ("i32", 16), + "stride_zn": ("i32", 1), + }, +) +@triton_autotune( + configs=get_mm_configs(), + key=["N", "K"], +) +@triton.jit +def _addmm_fwd( + x_ptr, + w_ptr, + y_ptr, + z_ptr, + M, + N, + K, + stride_xm, + stride_xk, + stride_wk, + stride_wn, + stride_ym, + stride_yn, + stride_zm, + stride_zn, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, +): + pid_0, pid_1 = tl.program_id(axis=0), tl.program_id(axis=1) + pid = pid_0 * tl.num_programs(axis=1) + pid_1 + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_m = tl.arange(0, BLOCK_M) + offs_k = tl.arange(0, BLOCK_K) + offs_n = tl.arange(0, BLOCK_N) + mask_m = (pid_m * BLOCK_M + offs_m)[:, None] < M + mask_n = (pid_n * BLOCK_N + offs_n)[None, :] < N + x_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_xm + x_ptrs = x_ptr + (offs_m[:, None] * stride_xm + offs_k[None, :] * stride_xk) + w_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_wn + w_ptrs = w_ptr + (offs_k[:, None] * stride_wk + offs_n[None, :] * stride_wn) + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(0, tl.cdiv(K, BLOCK_K)): + mask_k = offs_k[None, :] < K - k * BLOCK_K + x = tl.load(x_ptrs, mask=mask_k & mask_m, other=0.0) + mask_k = offs_k[:, None] < K - k * BLOCK_K + w = tl.load(w_ptrs, mask=mask_k & mask_n, other=0.0) + accumulator += tl.dot(x, w, allow_tf32=ALLOW_TF32) + x_ptrs += BLOCK_K * stride_xk + w_ptrs += BLOCK_K * stride_wk + + z_mask = mask_m & mask_n + if BROADCAST_Y: + # y is a vector, broadcast to add to z + y_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_yn + y_ptrs = y_ptr + stride_yn * offs_n[None, :] + y = tl.load(y_ptrs, mask=mask_n) + else: + y_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_ym + y_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_yn + y_ptrs = y_ptr + stride_ym * offs_m[:, None] + stride_yn * offs_n[None, :] + y = tl.load(y_ptrs, mask=z_mask) + z = (accumulator + y.to(tl.float32)).to(z_ptr.dtype.element_ty) + z_ptr += pid_m.to(tl.int64) * BLOCK_M * stride_zm + z_ptr += pid_n.to(tl.int64) * BLOCK_N * stride_zn + z_ptrs = z_ptr + stride_zm * offs_m[:, None] + stride_zn * offs_n[None, :] + tl.store(z_ptrs, z, mask=z_mask) + + +def _addmm_tma_set_block_size_hook(nargs): + BLOCK_M = nargs["BLOCK_M"] + BLOCK_N = nargs["BLOCK_N"] + BLOCK_K = nargs["BLOCK_K"] + NUM_MMA_GROUPS = nargs.get("NUM_MMA_GROUPS", 1) + BLOCK_M_SPLIT = BLOCK_M // NUM_MMA_GROUPS + PAIR_CTA = nargs.get("PAIR_CTA", False) + nargs["x_desc"].block_shape = [BLOCK_M_SPLIT, BLOCK_K] + # In PAIR_CTA mode, each CTA loads BLOCK_N // 2 of W + if PAIR_CTA: + nargs["w_desc"].block_shape = [BLOCK_K, BLOCK_N // 2] + else: + nargs["w_desc"].block_shape = [BLOCK_K, BLOCK_N] + EPILOGUE_SUBTILE = nargs.get("EPILOGUE_SUBTILE", 1) + nargs["z_desc"].block_shape = [BLOCK_M_SPLIT, BLOCK_N // EPILOGUE_SUBTILE] + if nargs["BROADCAST_Y"]: + nargs["y_desc"].block_shape = [1, BLOCK_N // EPILOGUE_SUBTILE] + else: + nargs["y_desc"].block_shape = [BLOCK_M_SPLIT, BLOCK_N // EPILOGUE_SUBTILE] + + +@triton.jit +def _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS): + group_id = tile_id // num_pid_in_group + first_pid_m = group_id * GROUP_SIZE_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) + pid_m = first_pid_m + (tile_id % group_size_m) + pid_n = (tile_id % num_pid_in_group) // group_size_m + return pid_m, pid_n + + +@triton.jit +def _addmm_persistent_tile_body( + x_desc, + w_desc, + y_desc, + z_desc, + tile_id, + num_pid_in_group, + num_pid_m, + k_tiles, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, + NUM_SMS: tl.constexpr, + EPILOGUE_SUBTILE: tl.constexpr, + INNER_WARP_SPECIALIZE: tl.constexpr, +): + pid_m, pid_n = _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS) + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in tl.range(0, k_tiles, warp_specialize=INNER_WARP_SPECIALIZE): + offs_k = k * BLOCK_K + x = x_desc.load([offs_xm, offs_k]) + w = w_desc.load([offs_k, offs_wn]) + accumulator = tl.dot(x, w, accumulator, allow_tf32=ALLOW_TF32) + + # Epilogue subtiling breaks the store into multiple pieces to reduce + # shared memory consumption and allow higher stage counts. + tl.static_assert( + EPILOGUE_SUBTILE <= 8, + "EPILOGUE_SUBTILE > 8 is not supported", + ) + acc_subtiles = _split_n_2D(accumulator, EPILOGUE_SUBTILE) # pyre-ignore[16] + slice_size: tl.constexpr = BLOCK_N // EPILOGUE_SUBTILE + for i in tl.static_range(EPILOGUE_SUBTILE): + if BROADCAST_Y: + y_i = y_desc.load([0, offs_wn + i * slice_size]) + else: + y_i = y_desc.load([offs_xm, offs_wn + i * slice_size]) + z_i = (acc_subtiles[i] + y_i.to(tl.float32)).to(z_desc.dtype) + z_desc.store([offs_xm, offs_wn + i * slice_size], z_i) + + +@triton_autotune( + configs=get_triton_persistent_configs(pre_hook=_addmm_tma_set_block_size_hook), + key=["M", "N", "K", "WARP_SPECIALIZE"], + prune_configs_by={"early_config_prune": _prune_persistent_autows_configs}, +) +@triton.jit +def _addmm_fwd_tma_persistent( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, + WARP_SPECIALIZE: tl.constexpr, + NUM_SMS: tl.constexpr, + EPILOGUE_SUBTILE: tl.constexpr, + DATA_PARTITION_FACTOR: tl.constexpr, + USE_META_WS: tl.constexpr, +): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + k_tiles = tl.cdiv(K, BLOCK_K) + num_tiles = num_pid_m * num_pid_n + + num_pid_in_group = GROUP_M * num_pid_n + + if USE_META_WS: + # Some arguments are only available in FBexperimental. + # pyre-ignore[28]: smem_alloc_algo is FBexperimental + for tile_id in tl.range( + start_pid, + num_tiles, + NUM_SMS, + flatten=False, + warp_specialize=WARP_SPECIALIZE, + data_partition_factor=DATA_PARTITION_FACTOR, + smem_alloc_algo=1, + ): + _addmm_persistent_tile_body( + x_desc, + w_desc, + y_desc, + z_desc, + tile_id, + num_pid_in_group, + num_pid_m, + k_tiles, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + BLOCK_K=BLOCK_K, + GROUP_M=GROUP_M, + ALLOW_TF32=ALLOW_TF32, + BROADCAST_Y=BROADCAST_Y, + NUM_SMS=NUM_SMS, + EPILOGUE_SUBTILE=EPILOGUE_SUBTILE, + INNER_WARP_SPECIALIZE=tl.constexpr(False), + ) + else: + # Pure OAI Triton version. + for tile_id in tl.range( + start_pid, num_tiles, NUM_SMS, flatten=True, warp_specialize=WARP_SPECIALIZE + ): + _addmm_persistent_tile_body( + x_desc, + w_desc, + y_desc, + z_desc, + tile_id, + num_pid_in_group, + num_pid_m, + k_tiles, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + BLOCK_K=BLOCK_K, + GROUP_M=GROUP_M, + ALLOW_TF32=ALLOW_TF32, + BROADCAST_Y=BROADCAST_Y, + NUM_SMS=NUM_SMS, + EPILOGUE_SUBTILE=EPILOGUE_SUBTILE, + INNER_WARP_SPECIALIZE=WARP_SPECIALIZE, + ) + + +@triton_autotune( + configs=get_mm_configs(pre_hook=_addmm_tma_set_block_size_hook), + key=["N", "K"], +) +@triton.jit +def _addmm_fwd_tma_ws( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, + NUM_SMEM_BUFFERS: tl.constexpr, +): + x_buffers = tlx.local_alloc((BLOCK_M, BLOCK_K), x_desc.dtype, NUM_SMEM_BUFFERS) + w_buffers = tlx.local_alloc((BLOCK_K, BLOCK_N), w_desc.dtype, NUM_SMEM_BUFFERS) + acc_tmem_buffer = tlx.local_alloc( + (BLOCK_M, BLOCK_N), tl.float32, tl.constexpr(1), tlx.storage_kind.tmem + ) + + if BROADCAST_Y: + y_buffer = tlx.local_alloc((1, BLOCK_N), y_desc.dtype, tl.constexpr(1)) + else: + y_buffer = tlx.local_alloc((BLOCK_M, BLOCK_N), y_desc.dtype, tl.constexpr(1)) + z_buffer = tlx.local_alloc((BLOCK_M, BLOCK_N), z_desc.dtype, tl.constexpr(1)) + + smem_full_bars = tlx.alloc_barriers(num_barriers=NUM_SMEM_BUFFERS, arrive_count=1) + smem_empty_bars = tlx.alloc_barriers(num_barriers=NUM_SMEM_BUFFERS, arrive_count=1) + y_load_barrier = tlx.alloc_barriers(num_barriers=1, arrive_count=1) + + with tlx.async_tasks(): + # Producer task: TMA loads + with tlx.async_task("default"): + pid_0, pid_1 = tl.program_id(axis=0), tl.program_id(axis=1) + pid = pid_0 * tl.num_programs(axis=1) + pid_1 + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + k_tiles = tl.cdiv(K, BLOCK_K) + + load_phase = 0 + for k in range(0, k_tiles): + buf = k % int(NUM_SMEM_BUFFERS) + + # Wait for buffer to be free + if k >= NUM_SMEM_BUFFERS: + tlx.barrier_wait(smem_empty_bars[buf], load_phase ^ 1) + + offs_k = k * BLOCK_K + tlx.barrier_expect_bytes( + smem_full_bars[buf], + 2 * (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N), + ) + tlx.async_descriptor_load( + x_desc, x_buffers[buf], [offs_xm, offs_k], smem_full_bars[buf] + ) + tlx.async_descriptor_load( + w_desc, w_buffers[buf], [offs_k, offs_wn], smem_full_bars[buf] + ) + + load_phase = load_phase ^ (buf == NUM_SMEM_BUFFERS - 1) + + # Consumer task: async_dot MMA + with tlx.async_task(num_warps=4, num_regs=232): + pid_0, pid_1 = tl.program_id(axis=0), tl.program_id(axis=1) + pid = pid_0 * tl.num_programs(axis=1) + pid_1 + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (pid % group_size_m) + pid_n = (pid % num_pid_in_group) // group_size_m + + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + k_tiles = tl.cdiv(K, BLOCK_K) + + # Start async load of y early + y_buf_view = tlx.local_view(y_buffer, 0) + y_load_bar = tlx.local_view(y_load_barrier, 0) + if BROADCAST_Y: + tlx.barrier_expect_bytes(y_load_bar, 1 * BLOCK_N * 2) + tlx.async_descriptor_load(y_desc, y_buf_view, [0, offs_wn], y_load_bar) + else: + tlx.barrier_expect_bytes(y_load_bar, BLOCK_M * BLOCK_N * 2) + tlx.async_descriptor_load( + y_desc, y_buf_view, [offs_xm, offs_wn], y_load_bar + ) + + dot_phase = 0 + for k in range(0, k_tiles): + buf = k % int(NUM_SMEM_BUFFERS) + tlx.barrier_wait(smem_full_bars[buf], dot_phase) + + tlx.async_dot( + x_buffers[buf], + w_buffers[buf], + acc_tmem_buffer[0], + use_acc=k > 0, + mBarriers=[smem_empty_bars[buf]], + out_dtype=tl.float32, + ) + + dot_phase = dot_phase ^ (buf == NUM_SMEM_BUFFERS - 1) + + last_buf = (k_tiles - 1) % NUM_SMEM_BUFFERS + last_dot_phase = dot_phase ^ (last_buf == NUM_SMEM_BUFFERS - 1) + tlx.barrier_wait(smem_empty_bars[last_buf], last_dot_phase) + + tmem_result = tlx.local_load(acc_tmem_buffer[0]) + + tlx.barrier_wait(y_load_bar, 0) + y = tlx.local_load(y_buf_view) + + z = (tmem_result + y.to(tl.float32)).to(z_desc.dtype) + z_buf_view = tlx.local_view(z_buffer, 0) + tlx.local_store(z_buf_view, z) + tlx.async_descriptor_store(z_desc, z_buf_view, [offs_xm, offs_wn]) + tlx.async_descriptor_store_wait(0) + + +@triton_autotune( + configs=_get_addmm_tma_ws_persistent_configs( + pre_hook=_addmm_tma_set_block_size_hook + ), + key=["M", "N", "K"], + prune_configs_by={"early_config_prune": _prune_configs_for_tlx_persistent_addmm}, +) +@triton.jit +def _addmm_fwd_tma_ws_persistent( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BROADCAST_Y: tl.constexpr, + NUM_SMEM_BUFFERS: tl.constexpr, + NUM_TMEM_BUFFERS: tl.constexpr, + NUM_SMS: tl.constexpr, + EPILOGUE_SUBTILE: tl.constexpr, + PAIR_CTA: tl.constexpr, + NUM_MMA_GROUPS: tl.constexpr, +): + BLOCK_M_SPLIT: tl.constexpr = BLOCK_M // NUM_MMA_GROUPS + + # Allocate buffers once for all tiles + x_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, BLOCK_K), x_desc.dtype, NUM_SMEM_BUFFERS * NUM_MMA_GROUPS + ) + # In pair CTA mode, each CTA only needs to load half of W + if PAIR_CTA: + w_buffers = tlx.local_alloc( + (BLOCK_K, BLOCK_N // 2), w_desc.dtype, NUM_SMEM_BUFFERS + ) + else: + w_buffers = tlx.local_alloc((BLOCK_K, BLOCK_N), w_desc.dtype, NUM_SMEM_BUFFERS) + tmem_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, BLOCK_N), + tl.float32, + NUM_TMEM_BUFFERS * NUM_MMA_GROUPS, + tlx.storage_kind.tmem, + ) + slice_size: tl.constexpr = BLOCK_N // EPILOGUE_SUBTILE + + Y_Z_SHARED: tl.constexpr = NUM_MMA_GROUPS == 2 and not BROADCAST_Y + if Y_Z_SHARED: + NUM_Z_BUFFERS: tl.constexpr = EPILOGUE_SUBTILE * NUM_MMA_GROUPS + else: + NUM_Z_BUFFERS: tl.constexpr = NUM_MMA_GROUPS + + if Y_Z_SHARED: + bias_storage_alias = tlx.storage_alias_spec(storage=tlx.storage_kind.smem) + y_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, slice_size), + y_desc.dtype, + NUM_Z_BUFFERS, + reuse=bias_storage_alias, + ) + z_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, slice_size), + z_desc.dtype, + NUM_Z_BUFFERS, + reuse=bias_storage_alias, + ) + # Define y and z to share a single buffer + bias_storage_alias.set_buffer_overlap( + tlx.reuse_group( + y_buffers, + z_buffers, + group_type=tlx.reuse_group_type.shared, + ) + ) + else: + if BROADCAST_Y: + y_buffers = tlx.local_alloc( + (1, slice_size), y_desc.dtype, EPILOGUE_SUBTILE * NUM_MMA_GROUPS + ) + else: + y_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, slice_size), + y_desc.dtype, + EPILOGUE_SUBTILE * NUM_MMA_GROUPS, + ) + z_buffers = tlx.local_alloc( + (BLOCK_M_SPLIT, slice_size), z_desc.dtype, NUM_Z_BUFFERS + ) + + cluster_cta_rank = tlx.cluster_cta_rank() + pred_cta0 = cluster_cta_rank == 0 + if PAIR_CTA: + cta_bars = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=2 + ) + + # Barriers for producer <-> MMA (separate X and W barriers) + x_smem_full_bars = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=1 + ) + x_smem_empty_bars = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=1 + ) + w_smem_full_bars = tlx.alloc_barriers(num_barriers=NUM_SMEM_BUFFERS, arrive_count=1) + # Barriers for MMA <-> Epilogue + tmem_full_bars = tlx.alloc_barriers( + num_barriers=NUM_TMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=1 + ) + tmem_empty_bars = tlx.alloc_barriers( + num_barriers=NUM_TMEM_BUFFERS * NUM_MMA_GROUPS, arrive_count=1 + ) + # Barriers for producer <-> Epilogue + # y_load_bar: producer signals when y data is ready + # y_empty_bar: epilogue signals when done using y buffer + y_load_bars = tlx.alloc_barriers( + num_barriers=EPILOGUE_SUBTILE * NUM_MMA_GROUPS, arrive_count=1 + ) + y_empty_bars = tlx.alloc_barriers( + num_barriers=EPILOGUE_SUBTILE * NUM_MMA_GROUPS, arrive_count=1 + ) + z_load_bars = tlx.alloc_barriers(num_barriers=NUM_Z_BUFFERS, arrive_count=1) + z_empty_bars = tlx.alloc_barriers(num_barriers=NUM_Z_BUFFERS, arrive_count=1) + + with tlx.async_tasks(): + # Epilogue consumer: waits for Y from producer, adds bias, stores to SMEM. + with tlx.async_task("default"): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + if PAIR_CTA: + # Round up to even for proper CTA pairing + num_pid_m = (num_pid_m + 1) // 2 * 2 + num_pid_n = tl.cdiv(N, BLOCK_N) + num_tiles = num_pid_m * num_pid_n + + tmem_read_phase = 0 + cur_tmem_buf = 0 + y_load_phase = 0 + z_load_phase = 0 + + z_idx = 0 + for _ in range(start_pid, num_tiles, NUM_SMS): + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + for group_id in tl.static_range(NUM_MMA_GROUPS): + buf_idx = group_id * NUM_TMEM_BUFFERS + cur_tmem_buf + acc_tmem = tmem_buffers[buf_idx] + if slice_id == 0: + # Wait for MMA to finish computing this group + tlx.barrier_wait(tmem_full_bars[buf_idx], tmem_read_phase) + + # Load result from TMEM and add bias + acc_subslice = tlx.subslice( + acc_tmem, slice_id * slice_size, slice_size + ) + result = tlx.local_load(acc_subslice) + if slice_id == EPILOGUE_SUBTILE - 1: + # Signal MMA that this TMEM buffer is now free + tlx.barrier_arrive(tmem_empty_bars[buf_idx], 1) + + y_idx = slice_id * NUM_MMA_GROUPS + group_id + y_buf_view = tlx.local_view(y_buffers, y_idx) + y_full = tlx.local_view(y_load_bars, y_idx) + tlx.barrier_wait(y_full, y_load_phase) + y = tlx.local_load(y_buf_view) + # If Y and Z are not shared signal we can load the next bias. + if not Y_Z_SHARED: + y_empty = tlx.local_view(y_empty_bars, y_idx) + tlx.barrier_arrive(y_empty, 1) + z = (result + y.to(tl.float32)).to(z_desc.dtype) + z_buf_view = tlx.local_view(z_buffers, z_idx) + # If Y and Z are not shared wait for Z to be empty. + # If there are shared this already guaranteed. + if not Y_Z_SHARED: + z_empty = tlx.local_view(z_empty_bars, z_idx) + tlx.barrier_wait(z_empty, z_load_phase ^ 1) + tlx.local_store(z_buf_view, z) + z_full = tlx.local_view(z_load_bars, z_idx) + tlx.barrier_arrive(z_full, 1) + z_load_phase = z_load_phase ^ (z_idx == (NUM_Z_BUFFERS - 1)) + # pyre-ignore[58] + z_idx = (z_idx + 1) % NUM_Z_BUFFERS + + tmem_read_phase = tmem_read_phase ^ ( + cur_tmem_buf == int(NUM_TMEM_BUFFERS) - 1 + ) + y_load_phase = y_load_phase ^ 1 + + cur_tmem_buf = (cur_tmem_buf + 1) % int(NUM_TMEM_BUFFERS) + + # MMA consumer: performs matrix multiplication + with tlx.async_task(num_warps=1, num_regs=24): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + if PAIR_CTA: + # Round up to even for proper CTA pairing + num_pid_m = (num_pid_m + 1) // 2 * 2 + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + k_tiles = tl.cdiv(K, BLOCK_K) + + dot_phase = 0 + tmem_write_phase = 1 + cur_tmem_buf = 0 + processed_k_iters = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + + # First K iteration (peeled): use_acc=False + buf = processed_k_iters % int(NUM_SMEM_BUFFERS) + tlx.barrier_wait(w_smem_full_bars[buf], dot_phase) + + for group_id in tl.static_range(NUM_MMA_GROUPS): + a_buf = group_id * NUM_SMEM_BUFFERS + buf + acc_buf = group_id * NUM_TMEM_BUFFERS + cur_tmem_buf + + tlx.barrier_wait(x_smem_full_bars[a_buf], dot_phase) + + # Wait for epilogue to finish with this TMEM buffer + tlx.barrier_wait(tmem_empty_bars[acc_buf], tmem_write_phase) + + if PAIR_CTA: + # pyre-ignore[61] + tlx.barrier_arrive(cta_bars[a_buf], 1, remote_cta_rank=0) + # pyre-ignore[61] + tlx.barrier_wait( + # pyre-ignore[61] + cta_bars[a_buf], + phase=dot_phase, + pred=pred_cta0, + ) + + tlx.async_dot( + x_buffers[a_buf], + w_buffers[buf], + tmem_buffers[acc_buf], + use_acc=False, + mBarriers=[x_smem_empty_bars[a_buf]], + two_ctas=PAIR_CTA, + out_dtype=tl.float32, + ) + + dot_phase = dot_phase ^ (buf == int(NUM_SMEM_BUFFERS) - 1) + + # Remaining K iterations: use_acc=True + for k in range(1, k_tiles): + buf = (processed_k_iters + k) % int(NUM_SMEM_BUFFERS) + tlx.barrier_wait(w_smem_full_bars[buf], dot_phase) + + for group_id in tl.static_range(NUM_MMA_GROUPS): + a_buf = group_id * NUM_SMEM_BUFFERS + buf + acc_buf = group_id * NUM_TMEM_BUFFERS + cur_tmem_buf + + tlx.barrier_wait(x_smem_full_bars[a_buf], dot_phase) + + if PAIR_CTA: + # pyre-ignore[61] + tlx.barrier_arrive(cta_bars[a_buf], 1, remote_cta_rank=0) + # pyre-ignore[61] + tlx.barrier_wait( + # pyre-ignore[61] + cta_bars[a_buf], + phase=dot_phase, + # pyre-ignore[61] + pred=pred_cta0, + ) + + tlx.async_dot( + x_buffers[a_buf], + w_buffers[buf], + tmem_buffers[acc_buf], + use_acc=True, + mBarriers=[x_smem_empty_bars[a_buf]], + two_ctas=PAIR_CTA, + out_dtype=tl.float32, + ) + + dot_phase = dot_phase ^ (buf == int(NUM_SMEM_BUFFERS) - 1) + + # Wait for last MMA to complete and signal epilogue + last_buf = (processed_k_iters + k_tiles - 1) % int(NUM_SMEM_BUFFERS) + last_dot_phase = dot_phase ^ (last_buf == int(NUM_SMEM_BUFFERS) - 1) + for group_id in tl.static_range(NUM_MMA_GROUPS): + a_buf = group_id * NUM_SMEM_BUFFERS + last_buf + tlx.barrier_wait(x_smem_empty_bars[a_buf], last_dot_phase) + acc_buf = group_id * NUM_TMEM_BUFFERS + cur_tmem_buf + # Signal epilogue that result is ready + tlx.barrier_arrive(tmem_full_bars[acc_buf], 1) + + tmem_write_phase = tmem_write_phase ^ ( + cur_tmem_buf == int(NUM_TMEM_BUFFERS) - 1 + ) + cur_tmem_buf = (cur_tmem_buf + 1) % int(NUM_TMEM_BUFFERS) + processed_k_iters += k_tiles + + # Producer: TMA loads for X, W, and Y + with tlx.async_task(num_warps=1, num_regs=24): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + if PAIR_CTA: + # Round up to even for proper CTA pairing + num_pid_m = (num_pid_m + 1) // 2 * 2 + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + k_tiles = tl.cdiv(K, BLOCK_K) + + load_phase = 0 + y_load_phase = 0 + processed_k_iters = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_xm = pid_m * BLOCK_M + # Full tile offset for y loading (both CTAs use same y) + offs_wn_full = pid_n * BLOCK_N + # Split W into two parts so each CTA has different offset + if PAIR_CTA: + # pyre-ignore[61] + offs_wn = pid_n * BLOCK_N + cluster_cta_rank * (BLOCK_N // 2) + else: + offs_wn = pid_n * BLOCK_N + + for k in range(0, k_tiles): + buf = (processed_k_iters + k) % int(NUM_SMEM_BUFFERS) + offs_k = k * BLOCK_K + + # Load X for group 0 + a_buf = buf # 0 * NUM_SMEM_BUFFERS + buf + tlx.barrier_wait(x_smem_empty_bars[a_buf], load_phase ^ 1) + tlx.barrier_expect_bytes( + x_smem_full_bars[a_buf], + 2 * BLOCK_M_SPLIT * BLOCK_K, + ) + tlx.async_descriptor_load( + x_desc, + x_buffers[a_buf], + [offs_xm, offs_k], + x_smem_full_bars[a_buf], + ) + + # Load W (wait for last group's x_empty to know W is free) + last_a_buf = (NUM_MMA_GROUPS - 1) * NUM_SMEM_BUFFERS + buf + tlx.barrier_wait(x_smem_empty_bars[last_a_buf], load_phase ^ 1) + if PAIR_CTA: + tlx.barrier_expect_bytes( + w_smem_full_bars[buf], + 2 * BLOCK_K * (BLOCK_N // 2), + ) + else: + tlx.barrier_expect_bytes( + w_smem_full_bars[buf], + 2 * BLOCK_K * BLOCK_N, + ) + tlx.async_descriptor_load( + w_desc, + w_buffers[buf], + [offs_k, offs_wn], + w_smem_full_bars[buf], + ) + + # Load X for remaining groups + for group_id in tl.static_range(1, NUM_MMA_GROUPS): + a_buf = group_id * NUM_SMEM_BUFFERS + buf + tlx.barrier_wait(x_smem_empty_bars[a_buf], load_phase ^ 1) + offs_xm2 = offs_xm + group_id * BLOCK_M_SPLIT + tlx.barrier_expect_bytes( + x_smem_full_bars[a_buf], + 2 * BLOCK_M_SPLIT * BLOCK_K, + ) + tlx.async_descriptor_load( + x_desc, + x_buffers[a_buf], + [offs_xm2, offs_k], + x_smem_full_bars[a_buf], + ) + + load_phase = load_phase ^ (buf == int(NUM_SMEM_BUFFERS) - 1) + + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + for group_id in tl.static_range(NUM_MMA_GROUPS): + y_idx = slice_id * NUM_MMA_GROUPS + group_id + y_buf_view = tlx.local_view(y_buffers, y_idx) + y_bar = tlx.local_view(y_load_bars, y_idx) + # If Y and Z are shared we need to wait for Z to be empty. + if Y_Z_SHARED: + y_empty = tlx.local_view(z_empty_bars, y_idx) + else: + y_empty = tlx.local_view(y_empty_bars, y_idx) + tlx.barrier_wait(y_empty, y_load_phase ^ 1) + if BROADCAST_Y: + tlx.barrier_expect_bytes(y_bar, 1 * slice_size * 2) + tlx.async_descriptor_load( + y_desc, + y_buf_view, + [0, offs_wn_full + slice_id * slice_size], + y_bar, + ) + else: + tlx.barrier_expect_bytes( + y_bar, BLOCK_M_SPLIT * slice_size * 2 + ) + tlx.async_descriptor_load( + y_desc, + y_buf_view, + [ + offs_xm + group_id * BLOCK_M_SPLIT, + offs_wn_full + slice_id * slice_size, + ], + y_bar, + ) + + y_load_phase = y_load_phase ^ 1 + + processed_k_iters += k_tiles + + # TMA Store consumer. Added to simplify the barrier + # logic. + with tlx.async_task(num_warps=1, num_regs=24): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + if PAIR_CTA: + # Round up to even for proper CTA pairing + num_pid_m = (num_pid_m + 1) // 2 * 2 + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + z_load_phase = 0 + + # Unroll the first iteration. + # This guraranteed safe from our grid size. + pid_m, pid_n = _compute_pid( + start_pid, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + z_idx = 0 + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + for group_id in tl.static_range(NUM_MMA_GROUPS): + # Determine the base "index" to decide if we need to wait on TMA. + z_idx_unrolled = slice_id * NUM_MMA_GROUPS + group_id + if z_idx_unrolled >= NUM_Z_BUFFERS: + tlx.async_descriptor_store_wait(NUM_Z_BUFFERS - 1) + z_empty = tlx.local_view(z_empty_bars, z_idx) + tlx.barrier_arrive(z_empty, 1) + + z_full = tlx.local_view(z_load_bars, z_idx) + tlx.barrier_wait(z_full, z_load_phase) + z_buf_view = tlx.local_view(z_buffers, z_idx) + tlx.fence_async_shared() + tlx.async_descriptor_store( + z_desc, + z_buf_view, + [ + offs_xm + group_id * BLOCK_M_SPLIT, + offs_wn + slice_id * slice_size, + ], + ) + + z_load_phase = z_load_phase ^ (z_idx == (NUM_Z_BUFFERS - 1)) + # pyre-ignore[58] + z_idx = (z_idx + 1) % NUM_Z_BUFFERS + + for tile_id in range(start_pid + NUM_SMS, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_xm = pid_m * BLOCK_M + offs_wn = pid_n * BLOCK_N + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + for group_id in tl.static_range(NUM_MMA_GROUPS): + # Wait on prior store to finish. + tlx.async_descriptor_store_wait(NUM_Z_BUFFERS - 1) + z_empty = tlx.local_view(z_empty_bars, z_idx) + tlx.barrier_arrive(z_empty, 1) + # Wait for the next load to be ready + z_full = tlx.local_view(z_load_bars, z_idx) + tlx.barrier_wait(z_full, z_load_phase) + z_buf_view = tlx.local_view(z_buffers, z_idx) + tlx.async_descriptor_store( + z_desc, + z_buf_view, + [ + offs_xm + group_id * BLOCK_M_SPLIT, + offs_wn + slice_id * slice_size, + ], + ) + z_load_phase = z_load_phase ^ (z_idx == (NUM_Z_BUFFERS - 1)) + # pyre-ignore[58] + z_idx = (z_idx + 1) % NUM_Z_BUFFERS + + # Wait for the last store. + tlx.async_descriptor_store_wait(0) + + +@torch.fx.wrap +def triton_addmm_fwd_tma_persistent( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, + warp_specialize: bool | None = None, +) -> torch.Tensor: + _meta_ws = _use_meta_ws() + if warp_specialize is None: + warp_specialize = _meta_ws + + M, K = x.shape + _, N = w.shape + + is_y_1d = y.dim() == 1 + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + # A dummy block value that will be overwritten when we have the real block size + dummy_block = [1, 1] + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + x_desc = TensorDescriptor(x, x.shape, x.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + w_desc = TensorDescriptor(w, w.shape, w.stride(), dummy_block) + y = y.reshape(1, -1) if is_y_1d else y + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + y_desc = TensorDescriptor(y, y.shape, y.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + z_desc = TensorDescriptor(z, z.shape, z.stride(), dummy_block) + NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count + + def grid(meta): + BLOCK_M = meta["BLOCK_M"] + BLOCK_N = meta["BLOCK_N"] + return ( + min( + NUM_SMS, + triton.cdiv(M, BLOCK_M) * triton.cdiv(N, BLOCK_N), + ), + ) + + _addmm_fwd_tma_persistent[grid]( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BROADCAST_Y=is_y_1d, + WARP_SPECIALIZE=warp_specialize, + NUM_SMS=NUM_SMS, + USE_META_WS=_meta_ws, + ) + return z + + +@torch.fx.wrap +def triton_addmm_fwd_tma_ws_tlx( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + M, K = x.shape + _, N = w.shape + + is_y_1d = y.dim() == 1 + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + # A dummy block value that will be overwritten when we have the real block size + dummy_block = [1, 1] + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + x_desc = TensorDescriptor(x, x.shape, x.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + w_desc = TensorDescriptor(w, w.shape, w.stride(), dummy_block) + y = y.reshape(1, -1) if is_y_1d else y + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + y_desc = TensorDescriptor(y, y.shape, y.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + z_desc = TensorDescriptor(z, z.shape, z.stride(), dummy_block) + + def grid(meta): + BLOCK_M = meta["BLOCK_M"] + BLOCK_N = meta["BLOCK_N"] + return ( + triton.cdiv(M, BLOCK_M), + triton.cdiv(N, BLOCK_N), + ) + + _addmm_fwd_tma_ws[grid]( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BROADCAST_Y=is_y_1d, + NUM_SMEM_BUFFERS=2, # Double buffering + ) + return z + + +@torch.fx.wrap +def triton_addmm_fwd_tma_ws_persistent_tlx( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + M, K = x.shape + _, N = w.shape + + is_y_1d = y.dim() == 1 + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count + + # A dummy block value that will be overwritten by the hook + dummy_block = [1, 1] + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + x_desc = TensorDescriptor(x, x.shape, x.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + w_desc = TensorDescriptor(w, w.shape, w.stride(), dummy_block) + y = y.reshape(1, -1) if is_y_1d else y + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + y_desc = TensorDescriptor(y, y.shape, y.stride(), dummy_block) + # pyre-ignore[6]: In call `TensorDescriptor.__init__`, for 2nd positional + # argument, expected `List[int]` but got `Size` + z_desc = TensorDescriptor(z, z.shape, z.stride(), dummy_block) + + def grid(meta): + BLOCK_M = meta["BLOCK_M"] + BLOCK_N = meta["BLOCK_N"] + num_pid_m = triton.cdiv(M, BLOCK_M) + num_pid_n = triton.cdiv(N, BLOCK_N) + # Round up num_pid_m to even for PAIR_CTA cluster compatibility + num_pid_m = (num_pid_m + 1) // 2 * 2 + total_tiles = num_pid_m * num_pid_n + grid_size = min(NUM_SMS, total_tiles) + # Ensure grid is even for cluster compatibility + if grid_size % 2 == 1: + grid_size = min(grid_size + 1, NUM_SMS) + # If rounding up exceeds NUM_SMS and NUM_SMS is odd, round down instead + if grid_size % 2 == 1: + grid_size = grid_size - 1 + return (grid_size,) + + _addmm_fwd_tma_ws_persistent[grid]( + x_desc, + w_desc, + y_desc, + z_desc, + M, + N, + K, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BROADCAST_Y=is_y_1d, + NUM_SMS=NUM_SMS, + ) + return z + + +@maybe_register_custom_op("generative_recommenders::triton_addmm_fwd", mutates_args=()) +def triton_addmm_fwd( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + M, K = x.shape + KB, N = w.shape + assert K == KB, f"incompatible dimensions {K}, {KB}" + + is_y_1d = y.dim() == 1 + NY = y.shape[0] if is_y_1d else y.shape[1] + assert N == NY, f"incompatible dimensions {N}, {NY}" + + # Allocate output + z = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return z + + grid = lambda meta: ( # noqa E731 + triton.cdiv(M, meta["BLOCK_M"]), + triton.cdiv(N, meta["BLOCK_N"]), + ) + + _addmm_fwd[grid]( + x, + w, + y, + z, + M, + N, + K, + x.stride(0), + x.stride(1), + w.stride(0), + w.stride(1), + y.stride(0) if not is_y_1d else 0, + y.stride(1) if not is_y_1d else y.stride(0), + z.stride(0), + z.stride(1), + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BROADCAST_Y=is_y_1d, + ) + return z + + +@triton_addmm_fwd.register_fake +def triton_addmm_fwd_fake( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + """Fake implementation for FakeTensor tracing.""" + M, _ = x.shape + _, N = w.shape + return torch.empty((M, N), device=x.device, dtype=x.dtype) + + +def triton_addmm_bwd( + x: torch.Tensor, + w: torch.Tensor, + dz: torch.Tensor, + is_y_1d: bool, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + if is_y_1d: + dy = torch.sum(dz, dim=0) + else: + dy = dz + dw = torch.mm(x.t(), dz) + dx = torch.mm(dz, w.t()) + + return dx, dw, dy + + +@maybe_register_custom_op( + "generative_recommenders::maybe_triton_addmm_fwd", mutates_args=() +) +def maybe_triton_addmm_fwd( + x: torch.Tensor, + w: torch.Tensor, + y: Optional[torch.Tensor], +) -> torch.Tensor: + # triton addmm is slower than torch (cublas) on AMD/Blackwell. + # Default to pytorch addmm on AMD/Blackwell for now. + if y is None: + return torch.mm(x, w) + if is_sm100_plus() or torch.version.hip is not None: + return torch.addmm(y, x, w) + else: + return triton_addmm_fwd(x=x, w=w, y=y) + + +@maybe_triton_addmm_fwd.register_fake +def maybe_triton_addmm_fwd_fake( + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, +) -> torch.Tensor: + """Fake implementation for FakeTensor tracing.""" + M, _ = x.shape + _, N = w.shape + return torch.empty((M, N), device=x.device, dtype=x.dtype) + + +class _AddMmFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + w: torch.Tensor, + y: torch.Tensor, + ) -> torch.Tensor: + ctx.save_for_backward(x, w) + ctx.is_y_1d = y.dim() == 1 + if is_sm100_plus() and TMA_AVAILABLE and _check_tma_alignment(x, w, y): + if x.dtype == torch.float32 or HAS_TLX == False: + return triton_addmm_fwd_tma_persistent(x, w, y, warp_specialize=True) + else: + return triton_addmm_fwd_tma_ws_persistent_tlx( + x, w, y + ) # tlx.async_dot doesn't support fp32 inputs because of WGMMA requirements + else: + return triton_addmm_fwd(x, w, y) + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dz: torch.Tensor + ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + (x, w) = ctx.saved_tensors + return triton_addmm_bwd(x, w, dz, ctx.is_y_1d) + + +def triton_addmm( + input: torch.Tensor, + mat1: torch.Tensor, + mat2: torch.Tensor, +) -> torch.Tensor: + return _AddMmFunction.apply(mat1, mat2, input) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_attention_utils.py b/recommendation_v4/generative_recommenders/ops/triton/triton_attention_utils.py new file mode 100644 index 000000000..61fd614f3 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_attention_utils.py @@ -0,0 +1,64 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl + + +@triton.jit +def acc_dq( + dq_ptrs_trans, + start_m, + stride_dqm, + k, + dqk_trans, + alpha, + mask_m, + MAX_SEQ_LEN, + LOCK, + BLOCK_M: tl.constexpr, + ATOMIC_ADD: tl.constexpr, + ALLOW_TF32: tl.constexpr, +): + if ATOMIC_ADD: + lock_id = start_m // BLOCK_M + stride_lock = tl.cdiv(MAX_SEQ_LEN, BLOCK_M) + lock = LOCK + tl.program_id(0) * stride_lock + lock_id + tl.debug_barrier() # add a barrier to force sync + while tl.atomic_cas(lock, 0, 1) == 1: + pass + dq_trans = tl.load( + dq_ptrs_trans + start_m * stride_dqm, + mask=mask_m[None, :], + other=0.0, + eviction_policy="evict_last", + ) + dq_trans += tl.dot(tl.trans(k), dqk_trans, allow_tf32=ALLOW_TF32) * alpha + dq_trans = dq_trans.to(k.dtype) + tl.store( + dq_ptrs_trans + start_m * stride_dqm, + dq_trans, + mask=mask_m[None, :], + eviction_policy="evict_last", + ) + if ATOMIC_ADD: + tl.atomic_xchg(lock, 0) # pyre-ignore [61] diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py new file mode 100644 index 000000000..768ef0013 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_attention.py @@ -0,0 +1,3242 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +#!/usr/bin/env python3 + +# pyre-unsafe + +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.ops.utils import ( + copy_if_different_ptr, + maybe_register_custom_op, +) + +try: + # @manual=//triton:triton + import triton.language.extra.tlx as tlx # type: ignore + + HAS_TLX = True +except ImportError: + # suppress type checking errors + tlx = None + + HAS_TLX = False + +from generative_recommenders.common import ( + autotune_max_seq_len, + prev_power_of_2, + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.triton._autotune_pinning import pinned_or_full +from triton.language.extra.libdevice import ( # @manual=//triton:triton + fast_dividef, + fast_expf, +) + +try: + # @manual=//triton:triton + from triton.tools.tensor_descriptor import TensorDescriptor + + tensor_descriptor_tma = True +except ImportError: + tensor_descriptor_tma = False + +try: + from generative_recommenders.ops.triton.fb.triton_attention_utils import acc_dq +except ImportError: + from generative_recommenders.ops.triton.triton_attention_utils import acc_dq + + +def _host_descriptor_pre_hook(nargs): + if not tensor_descriptor_tma: + return + + if not isinstance(nargs["Q"], TensorDescriptor): + return + BLOCK_M = nargs["BLOCK_M"] + BLOCK_N = nargs["BLOCK_N"] + BLOCK_D_Q = nargs["BLOCK_D_Q"] + BLOCK_D_V = nargs["BLOCK_D_V"] + if "USE_TLX" in nargs and nargs["USE_TLX"]: + BLOCK_M = BLOCK_M // nargs["NUM_MMA_GROUPS"] + nargs["Q"].block_shape = [BLOCK_M, BLOCK_D_Q] + nargs["V"].block_shape = [BLOCK_N, BLOCK_D_V] + nargs["K"].block_shape = [BLOCK_N, BLOCK_D_Q] + + +# pyre-ignore[2] +def _early_config_prune( + configs: List[triton.Config], + named_args, + **kwargs, +) -> List[triton.Config]: + """Filter autotune configs that are incompatible with the current call. + + The TLX (warp-specialized) variant of ``_hstu_attn_fwd`` calls + ``tlx.async_descriptor_load(Q, ...)`` which requires Q/K/V to be real TMA + tensor descriptors (``tl.tensor_descriptor_base``). They are only + constructed by the host wrapper when ``ENABLE_TMA=True`` AND the host + ``TensorDescriptor`` API is importable. If the kernel is invoked without + those preconditions, raw tensors flow into the TLX path and the + ``isinstance(desc, tl.tensor_descriptor_base)`` assert in + ``triton/language/extra/tlx/mem_ops.py`` fires at compile time. + + We make autotuning robust to that mismatch by dropping any config with + ``USE_TLX=True`` whenever ENABLE_TMA is not set or TMA host descriptors + are unavailable. This is purely defensive: if the caller threads + ``enable_tma=True`` (see ``_should_enable_tma`` below) the TLX configs + remain eligible. + """ + enable_tma = kwargs.get("ENABLE_TMA", None) + if enable_tma is None: + enable_tma = named_args.get("ENABLE_TMA", False) + if enable_tma and tensor_descriptor_tma: + return configs + pruned = [c for c in configs if not c.kwargs.get("USE_TLX", False)] + # Safety: never return an empty config list. + return pruned if pruned else configs + + +def _should_enable_tma() -> bool: + """Return True iff the TMA / TLX fast path can be safely enabled. + + Conditions: + * The host ``triton.tools.tensor_descriptor.TensorDescriptor`` API is + importable (``tensor_descriptor_tma``). + * CUDA is available and the device is Hopper (compute capability 9), + which is the only architecture for which TLX configs are emitted in + ``_get_fw_configs``. + """ + if not tensor_descriptor_tma: + return False + if not torch.cuda.is_available(): + return False + # NVIDIA-only gate: TMA (Tensor Memory Accelerator) is Hopper-specific + # hardware. On ROCm/HIP, `torch.cuda.get_device_capability()` mirrors the + # gfx name into a major.minor tuple — gfx950 (MI350X) returns (9, 5), which + # would otherwise pass the `device_capability == 9` check below and trick + # the kernel into taking the TMA path. The TMA path uses + # `triton.tools.tensor_descriptor.TensorDescriptor` and `TensorDescriptor.load` + # which lower to PTX `cp.async.bulk.tensor.*`; on AMD this either fails to + # compile or produces a kernel with mismatched reduction-dim shapes for + # `tl.dot(silu, v)` in `_hstu_attn_fwd_one_block` (see WARNING in + # `_hstu_attn_fwd` for the cascade). Bail out early on HIP so the + # non-TMA path is selected and AMD gets a working kernel. + if torch.version.hip: + return False + try: + device_capability = torch.cuda.get_device_capability()[0] + except (RuntimeError, AssertionError): + return False + return device_capability == 9 + + +def _get_fw_configs() -> List[triton.Config]: # noqa: C901 + configs = [] + if torch.version.hip: + for BLOCK_M in [32, 64, 128]: + for BLOCK_N in [32, 64]: + for num_stages in [1, 2]: + for num_warps in [4, 8]: + for matrix_instr_nonkdim in [16, 32]: + configs.append( + triton.Config( + { + "BLOCK_M": BLOCK_M, + "BLOCK_N": BLOCK_N, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "waves_per_eu": 0, + "kpack": 2, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + else: + configs = [ + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32}, + num_stages=2, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32}, + num_stages=2, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32}, + num_stages=4, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64}, + num_stages=4, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32}, + num_stages=4, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64}, + num_stages=2, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64}, + num_stages=4, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=2, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=4, + num_warps=2, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 32}, + num_stages=4, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64}, + num_stages=2, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64}, + num_stages=4, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 128}, + num_stages=4, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 128}, + num_stages=2, + num_warps=8, + pre_hook=_host_descriptor_pre_hook, + ), + ] + + # The `_hstu_attn_fwd` kernel signature unconditionally declares the four + # constexprs `USE_TLX`, `NUM_BUFFERS`, `NUM_MMA_WARPS_PER_GROUP`, + # `NUM_MMA_GROUPS` (introduced for the Hopper TLX warp-specialized variant). + # Triton requires every constexpr be bound at autotune time; missing any one + # of them triggers `TypeError: dynamic_func() missing N required positional + # arguments` during kernel dispatch. This loop populates the non-TLX defaults + # so the kernel call site doesn't have to know about TLX at all. + # + # IMPORTANT: this loop must apply to BOTH the HIP branch and the CUDA branch + # above. It used to live inside the CUDA `else:` block which meant HIP + # configs reached `_hstu_attn_fwd[grid](...)` without these defaults and + # crashed at dispatch. Keep this hoisted (outside the if/else) when + # editing — see commit message for the symptom. + for config in configs: + if not config.kwargs.get("USE_TLX", False): + config.kwargs["USE_TLX"] = False + config.kwargs["NUM_BUFFERS"] = 1 + config.kwargs["NUM_MMA_WARPS_PER_GROUP"] = 1 + config.kwargs["NUM_MMA_GROUPS"] = 1 + + # TLX (Triton Language Extension) warp-specialized configs are Hopper-only. + # Guard with `not torch.version.hip` so AMD never sees them — the TLX code + # path inside `_hstu_attn_fwd` calls `tlx.async_descriptor_load(...)` which + # requires real TMA tensor descriptors and only compiles on CUDA. + if not torch.version.hip: + if HAS_TLX: + try: + device_capability = torch.cuda.get_device_capability()[0] + except (RuntimeError, AssertionError): + # No CUDA device available + device_capability = None + + if device_capability == 9: + # H100 configs + configs.append( + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "USE_TLX": True, + "NUM_BUFFERS": 2, + "NUM_MMA_WARPS_PER_GROUP": 4, + "NUM_MMA_GROUPS": 2, + }, + num_stages=0, + num_warps=4, + pre_hook=_host_descriptor_pre_hook, + ), + ) + + return configs + + +@triton.jit +def _hstu_attn_fwd_one_block( # noqa: C901 + start_n, + seq_len, + offs_m, + offs_n, + q, + K, + V, + K_block_ptr, + V_block_ptr, + offset_kh, + offset_vh, + seq_start, + n_targets, + alpha, + MAX_SEQ_LEN, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_N: tl.constexpr, + ENABLE_TMA: tl.constexpr, +): + start_n = tl.multiple_of(start_n, BLOCK_N) + # -- compute qk ---- + k = None + qk = None + if ENABLE_TMA: + k = K.load( + [(seq_start + start_n).to(tl.int32), offset_kh.to(tl.int32)], + ) + # tma can only be loaded in one order, use trans afterwards + qk = tl.dot(q, tl.trans(k), allow_tf32=ALLOW_TF32) * alpha + else: + k = tl.load(K_block_ptr, boundary_check=(1,), padding_option="zero") + qk = tl.dot(q, k, allow_tf32=ALLOW_TF32) * alpha + invalid_mask = offs_m[:, None] == offs_n[None, :] + max_ids = seq_len + if HAS_CONTEXTUAL_SEQ_LEN: + offs_m = offs_m - contextual_seq_len + 1 + offs_m = tl.where( + offs_m > 0, + offs_m, + 0, + ) + offs_n = offs_n - contextual_seq_len + 1 + offs_n = tl.where( + offs_n > 0, + offs_n, + 0, + ) + max_ids = max_ids - contextual_seq_len + 1 + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + offs_m = tl.where( + offs_m < max_ids, + offs_m, + max_ids, + ) + offs_n = tl.where( + offs_n < max_ids, + offs_n, + max_ids, + ) + offs_m_minus_n = offs_m[:, None] - offs_n[None, :] + invalid_mask = invalid_mask or (offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask = invalid_mask and offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask = invalid_mask or ( + offs_m[:, None] == 0 and offs_n[None, :] < max_ids + ) + scale = tl.where(invalid_mask, (1.0 / MAX_SEQ_LEN), 0.0) + silu = fast_dividef(qk, 1.0 + fast_expf(-qk)) * scale + v = None + if ENABLE_TMA: + v = V.load( + [(seq_start + start_n).to(tl.int32), offset_vh.to(tl.int32)], + ) + else: + v = tl.load(V_block_ptr, boundary_check=(0,), padding_option="zero") + silu = silu.to(v.dtype) + return tl.dot(silu, v, allow_tf32=ALLOW_TF32) + + +@triton.jit +def _hstu_attn_fwd_compute( # noqa C901 + Q, + K, + V, + H, + DimQ, + DimV, + workspace_ptr, + seq_offsets, + num_targets, + Out, + stride_qm, + stride_qh, + stride_kn, + stride_kh, + stride_vn, + stride_vh, + stride_om, + stride_oh, + alpha, + MAX_SEQ_LEN, + DeltaSize, + contextual_seq_len, + max_attn_len, + off_z, + off_h, + pid, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ENABLE_TMA: tl.constexpr, + TMA_DESC_SIZE: tl.constexpr, +): + seq_start = tl.load(seq_offsets + off_z).to(tl.int64) + off_h = off_h.to(tl.int64) + off_z = off_z.to(tl.int64) + seq_end = tl.load(seq_offsets + off_z + 1) + seq_len = (seq_end - seq_start).to(tl.int32) + + if IS_DELTA_Q: + start_m_delta = pid * BLOCK_M + start_m = (start_m_delta + seq_len - DeltaSize).to(tl.int32) + else: + start_m_delta = 0 + start_m = pid * BLOCK_M + if start_m < seq_len: + if HAS_MULTIPLE_TARGETS: + n_targets = tl.load(num_targets + off_z).to(tl.int32) + else: + n_targets = None + + # initialize offsets + offs_m = start_m + tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + Q_block_ptr = None + K_block_ptr = None + V_block_ptr = None + if not ENABLE_TMA: + if IS_DELTA_Q: + Q_block_ptr = tl.make_block_ptr( + base=Q + off_h * stride_qh + off_z * DeltaSize * stride_qm, + shape=(DeltaSize, BLOCK_D_Q), + strides=(stride_qm, 1), + offsets=(start_m_delta, 0), + block_shape=(BLOCK_M, BLOCK_D_Q), + order=(1, 0), + ) + else: + Q_block_ptr = tl.make_block_ptr( + base=Q + off_h * stride_qh + seq_start * stride_qm, + shape=(seq_len, BLOCK_D_Q), + strides=(stride_qm, 1), + offsets=(start_m, 0), + block_shape=(BLOCK_M, BLOCK_D_Q), + order=(1, 0), + ) + q = tl.load(Q_block_ptr, boundary_check=(0,), padding_option="zero") + + K_block_ptr = tl.make_block_ptr( + base=K + off_h * stride_kh + seq_start * stride_kn, + shape=(BLOCK_D_Q, seq_len), + strides=(1, stride_kn), + offsets=(0, 0), + block_shape=(BLOCK_D_Q, BLOCK_N), + order=(0, 1), + ) + V_block_ptr = tl.make_block_ptr( + base=V + off_h * stride_vh + seq_start * stride_vn, + shape=(seq_len, BLOCK_D_V), + strides=(stride_vn, 1), + offsets=(0, 0), + block_shape=(BLOCK_N, BLOCK_D_V), + order=(1, 0), + ) + else: + if IS_DELTA_Q: + q = Q.load( + [ + (off_z * DeltaSize + start_m_delta).to(tl.int32), + (off_h * stride_qh).to(tl.int32), + ] + ) + else: + q = Q.load( + [ + (seq_start + start_m).to(tl.int32), + (off_h * stride_qh).to(tl.int32), + ] + ) + + acc = tl.zeros([BLOCK_M, BLOCK_D_V], dtype=tl.float32) + if HAS_MULTIPLE_TARGETS: + uih_end = seq_len - n_targets + else: + uih_end = seq_len + if HAS_CONTEXTUAL_SEQ_LEN is True and start_m < contextual_seq_len: + # uih_end must be larger than start_m + low = 0 + high = seq_len + else: + low = 0 + high = start_m + BLOCK_M + if HAS_MAX_ATTN_LEN: + if start_m > uih_end: + low = uih_end - max_attn_len + else: + low = start_m - max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + low = low if low > contextual_seq_len else 0 + else: + low = low if low > 0 else 0 + if HAS_MULTIPLE_TARGETS: + uih_end = (uih_end + BLOCK_N - 1) // BLOCK_N * BLOCK_N + if uih_end < start_m: + high = seq_len - n_targets + + if low > 0: + if not ENABLE_TMA: + K_block_ptr = tl.advance(K_block_ptr, (0, low)) + V_block_ptr = tl.advance(V_block_ptr, (low, 0)) + end_n = low + for start_n in range(low, high, BLOCK_N): + acc += _hstu_attn_fwd_one_block( + start_n=start_n, + seq_len=seq_len, + offs_m=offs_m, + offs_n=offs_n + start_n, + q=q, + K=K, + V=V, + K_block_ptr=K_block_ptr, + V_block_ptr=V_block_ptr, + offset_kh=off_h * stride_kh, + offset_vh=off_h * stride_vh, + seq_start=seq_start, + n_targets=n_targets if HAS_MULTIPLE_TARGETS else None, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_N=BLOCK_N, + ENABLE_TMA=ENABLE_TMA, + ) + if not ENABLE_TMA: + K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N)) + V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0)) + end_n += BLOCK_N + + if HAS_MULTIPLE_TARGETS: + # pyre-ignore[61] + if uih_end < start_m: + low_delta = start_m + high_delta = start_m + BLOCK_M + offset = (low_delta - end_n).to(tl.int32) + if not ENABLE_TMA: + K_block_ptr = tl.advance(K_block_ptr, (0, offset)) + V_block_ptr = tl.advance(V_block_ptr, (offset, 0)) + for start_delta in tl.range( + low_delta, high_delta, BLOCK_N, num_stages=0 + ): + acc += _hstu_attn_fwd_one_block( + start_n=start_delta, + seq_len=seq_len, + offs_m=offs_m, + offs_n=offs_n + start_delta, + q=q, + K=K, + V=V, + K_block_ptr=K_block_ptr, + V_block_ptr=V_block_ptr, + offset_kh=off_h * stride_kh, + offset_vh=off_h * stride_vh, + seq_start=seq_start, + n_targets=n_targets if HAS_MULTIPLE_TARGETS else None, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_N=BLOCK_N, + ENABLE_TMA=ENABLE_TMA, + ) + if not ENABLE_TMA: + K_block_ptr = tl.advance(K_block_ptr, (0, BLOCK_N)) + V_block_ptr = tl.advance(V_block_ptr, (BLOCK_N, 0)) + # Don't use TMA in Jagged case since we don't want to overwrite + # the output of another sequence + if IS_DELTA_Q: + start_m_delta = pid * BLOCK_M + offs_m_delta = start_m_delta + tl.arange(0, BLOCK_M) + offs_v_d = tl.arange(0, BLOCK_D_V) + off_o = Out + off_z * DeltaSize * stride_om + off_h * stride_oh + out_ptrs = off_o + offs_m_delta[:, None] * stride_om + offs_v_d[None, :] + tl.store(out_ptrs, acc, mask=(offs_m_delta < DeltaSize)[:, None]) + else: + # rematerialize offsets to save registers + start_m = pid * BLOCK_M + offs_m = start_m + tl.arange(0, BLOCK_M) + offs_v_d = tl.arange(0, BLOCK_D_V) + off_o = Out + seq_start * stride_om + off_h * stride_oh + out_ptrs = off_o + offs_m[:, None] * stride_om + offs_v_d[None, :] + tl.store(out_ptrs, acc, mask=(offs_m < seq_len)[:, None]) + + +@triton.jit +def _hstu_attn_fwd_compute_main_loop_tlx( # noqa C901 + low, + high, + seq_len, + offs_m, + offs_n, + acc, + q_tiles, + k_tiles, + v_tiles, + q_fulls, + k_fulls, + v_fulls, + k_empties, + v_empties, + v_dtype, + n_targets, + alpha, + end_n, + loop_trip_cnt, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + cid: tl.constexpr, + BLOCK_N: tl.constexpr, + NUM_BUFFERS: tl.constexpr, + MAX_SEQ_LEN: tl.constexpr, + WAIT_FOR_Q: tl.constexpr, +): + if WAIT_FOR_Q: + # wait for the Q buffer to be populated by the producer + q_full = tlx.local_view(q_fulls, cid) + tlx.barrier_wait(q_full, 0) + + q_tile = tlx.local_view(q_tiles, cid) + + for start in tl.range(low + BLOCK_N, high, BLOCK_N, num_stages=0): + buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + kv_phase = (loop_trip_cnt // NUM_BUFFERS) % 2 + + start_n = tl.multiple_of(start, BLOCK_N) + offs_n_start = offs_n + offs_n = offs_n_start + start_n + + # wait for the K buffer to be populated by the producer + k_full = tlx.local_view(k_fulls, buf_id) + tlx.barrier_wait(k_full, kv_phase) + k_tile = tlx.local_view(k_tiles, buf_id) + + # tma can only be loaded in one order, use trans afterwards + k_tile = tlx.local_trans(k_tile) + # second + qk = tlx.async_dot(q_tile, k_tile) + # wait for the MMA using to complete + qk = tlx.async_dot_wait(0, qk) + # release the K buffer + k_empty = tlx.local_view(k_empties, buf_id) + tlx.barrier_arrive(k_empty, 1) + + qk = qk * alpha + + invalid_mask = offs_m[:, None] == offs_n[None, :] + max_ids = seq_len + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + offs_m = tl.where( + offs_m < max_ids, + offs_m, + max_ids, + ) + offs_n = tl.where( + offs_n < max_ids, + offs_n, + max_ids, + ) + offs_m_minus_n = offs_m[:, None] - offs_n[None, :] + invalid_mask = invalid_mask or (offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask = invalid_mask and offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask = invalid_mask or ( + offs_m[:, None] == 0 and offs_n[None, :] < max_ids + ) + scale = tl.where(invalid_mask, (1.0 / MAX_SEQ_LEN), 0.0) + silu = fast_dividef(qk, 1.0 + fast_expf(-qk)) * scale + silu = silu.to(v_dtype) + + # wait for the V buffer to be populated by the producer + v_full = tlx.local_view(v_fulls, buf_id) + tlx.barrier_wait(v_full, kv_phase) + v_tile = tlx.local_view(v_tiles, buf_id) + acc = tlx.async_dot(silu, v_tile, acc) + # wait for the MMA using to complete + acc = tlx.async_dot_wait(0, acc) + # release the V buffer + v_empty = tlx.local_view(v_empties, buf_id) + tlx.barrier_arrive(v_empty, 1) + + end_n += BLOCK_N + + # increment loop trip counts + loop_trip_cnt += 1 + + return acc, end_n, loop_trip_cnt + + +@triton.jit +def _hstu_attn_fwd_compute_main_loop_tlx_pipelined( # noqa C901 + low, + high, + seq_len, + offs_m, + offs_n, + acc, + q_tiles, + k_tiles, + v_tiles, + q_fulls, + k_fulls, + v_fulls, + k_empties, + v_empties, + v_dtype, + n_targets, + alpha, + end_n, + loop_trip_cnt, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + cid: tl.constexpr, + BLOCK_N: tl.constexpr, + NUM_BUFFERS: tl.constexpr, + MAX_SEQ_LEN: tl.constexpr, + WAIT_FOR_Q: tl.constexpr, +): + if WAIT_FOR_Q: + # wait for the Q buffer to be populated by the producer + q_full = tlx.local_view(q_fulls, cid) + tlx.barrier_wait(q_full, 0) + q_tile = tlx.local_view(q_tiles, cid) + + # wait for the K buffer to be populated by the producer + k_buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + k_phase = (loop_trip_cnt // NUM_BUFFERS) % 2 + + k_full = tlx.local_view(k_fulls, k_buf_id) + tlx.barrier_wait(k_full, k_phase) + k_tile = tlx.local_view(k_tiles, k_buf_id) + + # tma can only be loaded in one order, use trans afterwards + k_tile = tlx.local_trans(k_tile) + + # Pingpong + if cid == 0: + # Consumer 0 waits for Consumer 1 to reach synchronization point at barrier 9. + tlx.named_barrier_wait(9, 256) + else: + # Consumer 1 signals its arrival at barrier 9. + tlx.named_barrier_arrive(9, 256) + # Then waits at barrier 10 until Consumer 0 finishes issuing its async_dot. + tlx.named_barrier_wait(10, 256) + + qk = tlx.async_dot(q_tile, k_tile) + + if cid == 0: + # After issuing async_dot, Consumer 0 signals barrier 10 to unblock Consumer 1. + tlx.named_barrier_arrive(10, 256) + + # wait for the MMA using to complete + qk = tlx.async_dot_wait(0, qk) + # release the K buffer + k_empty = tlx.local_view(k_empties, k_buf_id) + tlx.barrier_arrive(k_empty, 1) + + qk = qk * alpha + + start_n = tl.multiple_of(low, BLOCK_N) + offs_n_start = offs_n + offs_n = offs_n_start + start_n + + invalid_mask = offs_m[:, None] == offs_n[None, :] + max_ids = seq_len + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + offs_m = tl.where( + offs_m < max_ids, + offs_m, + max_ids, + ) + offs_n = tl.where( + offs_n < max_ids, + offs_n, + max_ids, + ) + offs_m_minus_n = offs_m[:, None] - offs_n[None, :] + invalid_mask = invalid_mask or (offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask = invalid_mask and offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask = invalid_mask or ( + offs_m[:, None] == 0 and offs_n[None, :] < max_ids + ) + scale = tl.where(invalid_mask, (1.0 / MAX_SEQ_LEN), 0.0) + silu = fast_dividef(qk, 1.0 + fast_expf(-qk)) * scale + silu = silu.to(v_dtype) + + loop_trip_cnt += 1 + + for start in tl.range(low + BLOCK_N, high, BLOCK_N, num_stages=0): + start_n = tl.multiple_of(start, BLOCK_N) + offs_n = offs_n_start + start_n + + k_buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + k_phase = k_phase ^ (k_buf_id == 0) + + # wait for the K buffer to be populated by the producer + k_full = tlx.local_view(k_fulls, k_buf_id) + tlx.barrier_wait(k_full, k_phase) + k_tile = tlx.local_view(k_tiles, k_buf_id) + + # tma can only be loaded in one order, use trans afterwards + k_tile = tlx.local_trans(k_tile) + + qk = tlx.async_dot(q_tile, k_tile) + # wait for the MMA using to complete + prev_silu = silu + + v_buf_id = (loop_trip_cnt - 1) % NUM_BUFFERS + # v_phase = v_phase ^ (v_buf_id == 0) + v_phase = ((loop_trip_cnt - 1) // NUM_BUFFERS) % 2 + v_full = tlx.local_view(v_fulls, v_buf_id) + tlx.barrier_wait(v_full, v_phase) + v_tile = tlx.local_view(v_tiles, v_buf_id) + acc = tlx.async_dot(prev_silu, v_tile, acc) + qk = tlx.async_dot_wait(1, qk) + + # release the K buffer + k_empty = tlx.local_view(k_empties, k_buf_id) + tlx.barrier_arrive(k_empty, 1) + + qk = qk * alpha + invalid_mask = offs_m[:, None] == offs_n[None, :] + max_ids = seq_len + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + offs_m = tl.where( + offs_m < max_ids, + offs_m, + max_ids, + ) + offs_n = tl.where( + offs_n < max_ids, + offs_n, + max_ids, + ) + offs_m_minus_n = offs_m[:, None] - offs_n[None, :] + invalid_mask = invalid_mask or (offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask = invalid_mask and offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask = invalid_mask or ( + offs_m[:, None] == 0 and offs_n[None, :] < max_ids + ) + scale = tl.where(invalid_mask, (1.0 / MAX_SEQ_LEN), 0.0) + silu = fast_dividef(qk, 1.0 + fast_expf(-qk)) * scale + silu = silu.to(v_dtype) + + acc = tlx.async_dot_wait(0, acc) + # release the V buffer + v_empty = tlx.local_view(v_empties, v_buf_id) + tlx.barrier_arrive(v_empty, 1) + + end_n += BLOCK_N + + # increment loop trip counts + loop_trip_cnt += 1 + # v_buf_id = loop_trip_cnt % NUM_BUFFERS + # v_phase = (loop_trip_cnt // NUM_BUFFERS) % 2 + + # wait for the V buffer to be populated by the producer + v_buf_id = (loop_trip_cnt - 1) % NUM_BUFFERS + v_phase = ((loop_trip_cnt - 1) // NUM_BUFFERS) % 2 + v_full = tlx.local_view(v_fulls, v_buf_id) + # tlx.barrier_wait(v_full, v_buf_id) + v_tile = tlx.local_view(v_tiles, v_buf_id) + tlx.barrier_wait(v_full, v_phase) + acc = tlx.async_dot(silu, v_tile, acc) + acc = tlx.async_dot_wait(0, acc) + # release the V buffer + v_empty = tlx.local_view(v_empties, v_buf_id) + tlx.barrier_arrive(v_empty, 1) + + return acc, end_n, loop_trip_cnt + + +@triton.jit +def _hstu_attn_fwd_load_K_or_V( + K, + k_tiles, + k_empties, + k_fulls, + buf_id, + k_phase, + start_n, + seq_start, + offset_kh, + BLOCK_D_Q: tl.constexpr, + BLOCK_N: tl.constexpr, +): + # wait for the K buffer to be released by the consumer + k_empty = tlx.local_view(k_empties, buf_id) + tlx.barrier_wait(k_empty, k_phase) + # load K + k_full = tlx.local_view(k_fulls, buf_id) + k_tile = tlx.local_view(k_tiles, buf_id) + tlx.barrier_expect_bytes(k_full, 2 * BLOCK_N * BLOCK_D_Q) # float16 + tlx.async_descriptor_load( + K, + k_tile, + [(seq_start + start_n).to(tl.int32), offset_kh.to(tl.int32)], + k_full, + ) + + +@triton.jit +def _hstu_attn_fwd_load_Q( + Q, + q_tiles, + q_fulls, + cid, + off_z, + off_h, + stride_qh, + start_m, + seq_start, + DeltaSize, + IS_DELTA_Q: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_M: tl.constexpr, +): + q_full = tlx.local_view(q_fulls, cid) + tlx.barrier_expect_bytes(q_full, 2 * BLOCK_M * BLOCK_D_Q) # float16 + q_tile = tlx.local_view(q_tiles, cid) + seq_offset = start_m + cid * BLOCK_M + if IS_DELTA_Q: + tlx.async_descriptor_load( + Q, + q_tile, + [ + (off_z * DeltaSize + start_m).to(tl.int32), + (off_h * stride_qh).to(tl.int32), + ], + q_full, + ) + else: + tlx.async_descriptor_load( + Q, + q_tile, + [ + (seq_start + seq_offset).to(tl.int32), + (off_h * stride_qh).to(tl.int32), + ], + q_full, + ) + + +@triton.jit +def _hstu_attn_fwd_caculate_range( + seq_len, + start_m, + n_targets, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, +): + if HAS_MULTIPLE_TARGETS: + uih_end = seq_len - n_targets + else: + uih_end = seq_len + + if HAS_CONTEXTUAL_SEQ_LEN is True and start_m < contextual_seq_len: + # uih_end must be larger than start_m + low = 0 + high = seq_len + else: + low = 0 + high = start_m + BLOCK_M + if HAS_MAX_ATTN_LEN: + if start_m > uih_end: + low = uih_end - max_attn_len + else: + low = start_m - max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + low = low if low > contextual_seq_len else 0 + else: + low = low if low > 0 else 0 + if HAS_MULTIPLE_TARGETS: + uih_end = (uih_end + BLOCK_N - 1) // BLOCK_N * BLOCK_N + if uih_end < start_m: + high = seq_len - n_targets + + return low, high, uih_end + + +@triton.jit +def _hstu_attn_fwd_load_Q_K_V( + Q, + K, + V, + q_tiles, + k_tiles, + v_tiles, + q_fulls, + k_fulls, + v_fulls, + k_empties, + v_empties, + stride_qh, + stride_kh, + stride_vh, + contextual_seq_len, + max_attn_len, + DeltaSize, + off_z, + off_h, + start_m, + seq_start, + seq_len, + n_targets, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + NUM_BUFFERS: tl.constexpr, + NUM_MMA_GROUPS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, +): + # load q: it will stay in SRAM throughout + BLOCK_M_SPLIT: tl.constexpr = BLOCK_M // NUM_MMA_GROUPS + + _hstu_attn_fwd_load_Q( + Q=Q, + q_tiles=q_tiles, + q_fulls=q_fulls, + cid=0, + off_z=off_z, + off_h=off_h, + stride_qh=stride_qh, + start_m=start_m, + seq_start=seq_start, + DeltaSize=DeltaSize, + IS_DELTA_Q=IS_DELTA_Q, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_M=BLOCK_M_SPLIT, + ) + + off_h = off_h.to(tl.int64) + off_z = off_z.to(tl.int64) + offset_kh = off_h * stride_kh + offset_vh = off_h * stride_vh + + low, high, uih_end = _hstu_attn_fwd_caculate_range( + seq_len, + start_m, + n_targets, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN, + BLOCK_M, + BLOCK_N, + ) + + kv_phase = 0 + loop_trip_cnt = 0 + + # pyre-ignore[58] + buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + kv_phase = kv_phase ^ (buf_id == 0) + + start_n = tl.multiple_of(low, BLOCK_N) + + _hstu_attn_fwd_load_K_or_V( + K, + k_tiles, + k_empties, + k_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_kh, + BLOCK_D_Q, + BLOCK_N, + ) + + for cid in tl.range(1, NUM_MMA_GROUPS, loop_unroll_factor=NUM_MMA_GROUPS - 1): + _hstu_attn_fwd_load_Q( + Q, + q_tiles, + q_fulls, + cid, + off_z, + off_h, + stride_qh, + start_m, + seq_start, + DeltaSize, + IS_DELTA_Q, + BLOCK_D_Q, + BLOCK_M_SPLIT, + ) + + _hstu_attn_fwd_load_K_or_V( + V, + v_tiles, + v_empties, + v_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_vh, + BLOCK_D_V, + BLOCK_N, + ) + + loop_trip_cnt += 1 + + for start in range(low + BLOCK_N, high, BLOCK_N): + # pyre-ignore[58] + buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + kv_phase = kv_phase ^ (buf_id == 0) + + start_n = tl.multiple_of(start, BLOCK_N) + + _hstu_attn_fwd_load_K_or_V( + K, + k_tiles, + k_empties, + k_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_kh, + BLOCK_D_Q, + BLOCK_N, + ) + + _hstu_attn_fwd_load_K_or_V( + V, + v_tiles, + v_empties, + v_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_vh, + BLOCK_D_V, + BLOCK_N, + ) + + # increment loop trip counts + loop_trip_cnt += 1 + + # pyre-ignore[61] + if uih_end < start_m: + low_delta = start_m + high_delta = start_m + BLOCK_M + for start_delta in tl.range(low_delta, high_delta, BLOCK_N, num_stages=0): + # pyre-ignore[58] + buf_id = loop_trip_cnt % NUM_BUFFERS + # buffers in a row share the same phase + kv_phase = kv_phase ^ (buf_id == 0) + + start_n = tl.multiple_of(start_delta, BLOCK_N) + + _hstu_attn_fwd_load_K_or_V( + K, + k_tiles, + k_empties, + k_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_kh, + BLOCK_D_Q, + BLOCK_N, + ) + + _hstu_attn_fwd_load_K_or_V( + V, + v_tiles, + v_empties, + v_fulls, + buf_id, + kv_phase, + start_n, + seq_start, + offset_vh, + BLOCK_D_V, + BLOCK_N, + ) + + # increment loop trip counts + loop_trip_cnt += 1 + + +@triton.jit +def _hstu_attn_fwd_compute_tlx( # noqa C901 + Q, + K, + V, + H, + DimQ, + DimV, + seq_offsets, + num_targets, + Out, + stride_qh, + stride_kh, + stride_vh, + stride_om, + stride_oh, + alpha, + MAX_SEQ_LEN, + DeltaSize, + contextual_seq_len, + max_attn_len, + off_z, + off_h, + pid, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + NUM_BUFFERS: tl.constexpr, # + NUM_MMA_WARPS_PER_GROUP: tl.constexpr, # + NUM_MMA_GROUPS: tl.constexpr, # + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, +): + seq_start = tl.load(seq_offsets + off_z).to(tl.int64) + seq_end = tl.load(seq_offsets + off_z + 1) + seq_len = (seq_end - seq_start).to(tl.int32) + + if IS_DELTA_Q: + start_m = pid * BLOCK_M + start_m = (start_m + seq_len - DeltaSize).to(tl.int32) + else: + start_m = pid * BLOCK_M + + if start_m >= seq_len: + return + + if HAS_MULTIPLE_TARGETS: + n_targets = tl.load(num_targets + off_z).to(tl.int32) + else: + n_targets = None + + BLOCK_M_SPLIT: tl.constexpr = BLOCK_M // NUM_MMA_GROUPS + # allocate buffers + q_tiles = tlx.local_alloc( + (BLOCK_M_SPLIT, BLOCK_D_Q), tlx.dtype_of(Q), NUM_MMA_GROUPS + ) + k_tiles = tlx.local_alloc((BLOCK_N, BLOCK_D_Q), tlx.dtype_of(K), NUM_BUFFERS) + v_tiles = tlx.local_alloc((BLOCK_N, BLOCK_D_V), tlx.dtype_of(V), NUM_BUFFERS) + + # allocate barriers + q_fulls = tlx.alloc_barriers(num_barriers=NUM_MMA_GROUPS, arrive_count=1) + k_empties = tlx.alloc_barriers( + num_barriers=NUM_BUFFERS, arrive_count=NUM_MMA_GROUPS + ) + k_fulls = tlx.alloc_barriers(num_barriers=NUM_BUFFERS, arrive_count=1) + v_empties = tlx.alloc_barriers( + num_barriers=NUM_BUFFERS, arrive_count=NUM_MMA_GROUPS + ) + v_fulls = tlx.alloc_barriers(num_barriers=NUM_BUFFERS, arrive_count=1) + + with tlx.async_tasks(): + # producer group + with tlx.async_task("default"): + _hstu_attn_fwd_load_Q_K_V( + Q=Q, + K=K, + V=V, + q_tiles=q_tiles, + k_tiles=k_tiles, + v_tiles=v_tiles, + q_fulls=q_fulls, + k_fulls=k_fulls, + v_fulls=v_fulls, + k_empties=k_empties, + v_empties=v_empties, + stride_qh=stride_qh, + stride_kh=stride_kh, + stride_vh=stride_vh, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + DeltaSize=DeltaSize, + off_z=off_z, + off_h=off_h, + start_m=start_m, + seq_start=seq_start, + seq_len=seq_len, + n_targets=n_targets, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + IS_DELTA_Q=IS_DELTA_Q, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + NUM_BUFFERS=NUM_BUFFERS, + NUM_MMA_GROUPS=NUM_MMA_GROUPS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ) + + # consumer groups + with tlx.async_task( + num_warps=NUM_MMA_WARPS_PER_GROUP, registers=232, replicate=NUM_MMA_GROUPS + ): + cid = tlx.async_task_replica_id() + acc = tl.zeros([BLOCK_M_SPLIT, BLOCK_D_V], dtype=tl.float32) + # initialize offsets + offs_m = start_m + tl.arange(0, BLOCK_M_SPLIT) + cid * BLOCK_M_SPLIT + offs_n = tl.arange(0, BLOCK_N) + + low, high, uih_end = _hstu_attn_fwd_caculate_range( + seq_len, + start_m, + n_targets, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN, + BLOCK_M, + BLOCK_N, + ) + + end_n = low + loop_trip_cnt = 0 + + acc, end_n, loop_trip_cnt = _hstu_attn_fwd_compute_main_loop_tlx_pipelined( + low=low, + high=high, + seq_len=seq_len, + offs_m=offs_m, + offs_n=offs_n, + acc=acc, + q_tiles=q_tiles, + k_tiles=k_tiles, + v_tiles=v_tiles, + q_fulls=q_fulls, + k_fulls=k_fulls, + v_fulls=v_fulls, + k_empties=k_empties, + v_empties=v_empties, + v_dtype=tlx.dtype_of(V), + n_targets=n_targets, + alpha=alpha, + end_n=end_n, + loop_trip_cnt=loop_trip_cnt, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + cid=cid, + BLOCK_N=BLOCK_N, + NUM_BUFFERS=NUM_BUFFERS, + MAX_SEQ_LEN=MAX_SEQ_LEN, + WAIT_FOR_Q=1, + ) + + # pyre-ignore[61] + if uih_end < start_m: + low_delta = start_m + high_delta = start_m + BLOCK_M + acc, end_n, loop_trip_cnt = _hstu_attn_fwd_compute_main_loop_tlx( + low=low_delta, + high=high_delta, + seq_len=seq_len, + offs_m=offs_m, + offs_n=offs_n, + acc=acc, + q_tiles=q_tiles, + k_tiles=k_tiles, + v_tiles=v_tiles, + q_fulls=q_fulls, + k_fulls=k_fulls, + v_fulls=v_fulls, + k_empties=k_empties, + v_empties=v_empties, + v_dtype=tlx.dtype_of(V), + n_targets=n_targets, + alpha=alpha, + end_n=end_n, + loop_trip_cnt=loop_trip_cnt, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + cid=cid, + BLOCK_N=BLOCK_N, + NUM_BUFFERS=NUM_BUFFERS, + MAX_SEQ_LEN=MAX_SEQ_LEN, + WAIT_FOR_Q=0, + ) + + # Don't use TMA in Jagged case since we don't want to overwrite + # the output of another sequence + if IS_DELTA_Q: + start_m_delta = pid * BLOCK_M + cid * BLOCK_M_SPLIT + offs_m_delta = start_m_delta + tl.arange(0, BLOCK_M_SPLIT) + offs_v_d = tl.arange(0, BLOCK_D_V) + off_o = Out + off_z * DeltaSize * stride_om + off_h * stride_oh + out_ptrs = off_o + offs_m_delta[:, None] * stride_om + offs_v_d[None, :] + tl.store(out_ptrs, acc, mask=(offs_m_delta < DeltaSize)[:, None]) + else: + # rematerialize offsets to save registers + start_m = pid * BLOCK_M + cid * BLOCK_M_SPLIT + offs_m = start_m + tl.arange(0, BLOCK_M_SPLIT) + offs_v_d = tl.arange(0, BLOCK_D_V) + off_o = Out + seq_start * stride_om + off_h * stride_oh + out_ptrs = off_o + offs_m[:, None] * stride_om + offs_v_d[None, :] + tl.store(out_ptrs, acc, mask=(offs_m < seq_len)[:, None]) + + +def _get_fw_pinned_configs() -> List[triton.Config]: + # Pinned forward-attention configs for MI350X gfx950. The full search is a + # coin flip between matrix_instr_nonkdim 16 (fast) and 32 (slow); pinning the + # known winners makes cold starts deterministic. See _autotune_pinning. + # + # This is a LIST, one entry per training shape we've tuned (currently just + # bs=1024). With >1 entry the autotuner still runs a tiny benchmark over only + # these candidates and caches the winner per `key` (AUTOTUNE_Z / H / + # AUTOTUNE_MAX_SEQ_LEN / DimQ / DimV / ...), so each batch size automatically + # picks its own config — same pattern as the layer-norm pins. + # + # TO ADD A NEW BATCH SIZE / SHAPE: + # 1. Run once with TRITON_FULL_AUTOTUNE=1 TRITON_PRINT_AUTOTUNING=1. + # 2. Grep the log for "best config selected:" under "_hstu_attn_fwd". + # 3. Append that config below (copy BLOCK_M/BLOCK_N/matrix_instr_nonkdim/ + # waves_per_eu/kpack/num_stages/num_warps verbatim). + # The four USE_TLX/NUM_* defaults below are required by the kernel signature + # (see the USE_TLX-default loop in _get_fw_configs); the pinned path bypasses + # that loop, so every pinned entry must set them explicitly. + if torch.version.hip: + return [ + # --- yambda bs=1024, L=2048 winner (from capture log) --- + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 32, + "matrix_instr_nonkdim": 16, + "waves_per_eu": 0, + "kpack": 2, + "USE_TLX": False, + "NUM_BUFFERS": 1, + "NUM_MMA_WARPS_PER_GROUP": 1, + "NUM_MMA_GROUPS": 1, + }, + num_stages=2, + num_warps=8, + ), + # --- add more (bs, L) winners here; see "TO ADD A NEW BATCH SIZE" --- + ] + return _get_fw_configs() + + +@triton_autotune( + configs=pinned_or_full(_get_fw_pinned_configs(), _get_fw_configs), + key=[ + "AUTOTUNE_Z", + "H", + "AUTOTUNE_MAX_SEQ_LEN", + "DimQ", + "DimV", + "DeltaSize", + "IS_DELTA_Q", + ], + prune_configs_by={"early_config_prune": _early_config_prune}, +) +@triton.jit +def _hstu_attn_fwd( # noqa C901 + Q, + K, + V, + workspace_ptr, + sort_by_length_indices, + seq_offsets, + num_targets, + Out, + stride_qm, + stride_qh, + stride_kn, + stride_kh, + stride_vn, + stride_vh, + stride_om, + stride_oh, + alpha, + Z, + AUTOTUNE_Z, + H, + MAX_SEQ_LEN, + AUTOTUNE_MAX_SEQ_LEN, # Quantized MAX_SEQ_LEN used as an autotuning key + DimQ, + DimV, + DeltaSize, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + USE_TLX: tl.constexpr, + NUM_BUFFERS: tl.constexpr, # + NUM_MMA_WARPS_PER_GROUP: tl.constexpr, # + NUM_MMA_GROUPS: tl.constexpr, # + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + HAS_SORT_BY_LENGTH_INDICES: tl.constexpr, + ENABLE_TMA: tl.constexpr, + TMA_DESC_SIZE: tl.constexpr, +): + off_hz = tl.program_id(1) + off_z = off_hz // H + if HAS_SORT_BY_LENGTH_INDICES: + off_z = tl.load(sort_by_length_indices + off_z) + off_h = off_hz % H + pid = tl.program_id(0) + if USE_TLX: + _hstu_attn_fwd_compute_tlx( + Q=Q, + K=K, + V=V, + H=H, + DimQ=DimQ, + DimV=DimV, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=Out, + stride_qh=stride_qh, + stride_kh=stride_kh, + stride_vh=stride_vh, + stride_om=stride_om, + stride_oh=stride_oh, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + DeltaSize=DeltaSize, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + off_z=off_z, + off_h=off_h, + pid=pid, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + IS_DELTA_Q=IS_DELTA_Q, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + NUM_BUFFERS=NUM_BUFFERS, + NUM_MMA_WARPS_PER_GROUP=NUM_MMA_WARPS_PER_GROUP, + NUM_MMA_GROUPS=NUM_MMA_GROUPS, + ) + else: + _hstu_attn_fwd_compute( + Q=Q, + K=K, + V=V, + H=H, + DimQ=DimQ, + DimV=DimV, + workspace_ptr=workspace_ptr, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=Out, + stride_qm=stride_qm, + stride_qh=stride_qh, + stride_kn=stride_kn, + stride_kh=stride_kh, + stride_vn=stride_vn, + stride_vh=stride_vh, + stride_om=stride_om, + stride_oh=stride_oh, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + DeltaSize=DeltaSize, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + off_z=off_z, + off_h=off_h, + pid=pid, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + IS_DELTA_Q=IS_DELTA_Q, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + ENABLE_TMA=ENABLE_TMA, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ) + + +@triton_autotune( + configs=pinned_or_full(_get_fw_pinned_configs(), _get_fw_configs), + key=[ + "AUTOTUNE_Z", + "H", + "AUTOTUNE_MAX_SEQ_LEN", + "DimQ", + "DimV", + "DeltaSize", + "IS_DELTA_Q", + ], + prune_configs_by={"early_config_prune": _early_config_prune}, +) +@triton.jit +def _hstu_attn_fwd_persistent( # noqa C901 + Q, + K, + V, + workspace_ptr, + sort_by_length_indices, + seq_offsets, + num_targets, + Out, + stride_qm, + stride_qh, + stride_kn, + stride_kh, + stride_vn, + stride_vh, + stride_om, + stride_oh, + alpha, + Z, + AUTOTUNE_Z, + H, + MAX_SEQ_LEN, + AUTOTUNE_MAX_SEQ_LEN, # Quantized MAX_SEQ_LEN used as an autotuning key + DimQ, + DimV, + DeltaSize, + contextual_seq_len, + max_attn_len, + HAS_MULTIPLE_TARGETS: tl.constexpr, + IS_DELTA_Q: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + USE_TLX: tl.constexpr, + NUM_BUFFERS: tl.constexpr, # + NUM_MMA_WARPS_PER_GROUP: tl.constexpr, # + NUM_MMA_GROUPS: tl.constexpr, # + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + HAS_SORT_BY_LENGTH_INDICES: tl.constexpr, + ENABLE_TMA: tl.constexpr, + TMA_DESC_SIZE: tl.constexpr, +): + n_tile_num = tl.cdiv(MAX_SEQ_LEN, BLOCK_M) + prog_id = tl.program_id(0) + num_progs = tl.num_programs(0) + + total_tiles = n_tile_num * Z * H + + tiles_per_sm = total_tiles // num_progs + if prog_id < total_tiles % num_progs: + tiles_per_sm += 1 + + tile_idx = prog_id + for _ in range(0, tiles_per_sm): + pid = (total_tiles - tile_idx - 1) // (Z * H) + off_hz = (total_tiles - tile_idx - 1) % (Z * H) + off_z = off_hz // H + off_h = off_hz % H + _hstu_attn_fwd_compute( + Q=Q, + K=K, + V=V, + H=H, + DimQ=DimQ, + DimV=DimV, + workspace_ptr=workspace_ptr, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=Out, + stride_qm=stride_qm, + stride_qh=stride_qh, + stride_kn=stride_kn, + stride_kh=stride_kh, + stride_vn=stride_vn, + stride_vh=stride_vh, + stride_om=stride_om, + stride_oh=stride_oh, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + DeltaSize=DeltaSize, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + off_z=off_z, + off_h=off_h, + pid=pid, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + IS_DELTA_Q=IS_DELTA_Q, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + ENABLE_TMA=ENABLE_TMA, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ) + tile_idx += num_progs + + +@triton.jit +def _hstu_attn_bwd_one_block( # noqa C901 + start_m, + offs_n, + offs_m, + q_ptrs_trans, + dq_ptrs_trans, + do_ptrs, + device_desc_q, + device_desc_do, + dk, + dv, + k, + v, + pos_offs_n, + seq_len, + max_ids, + contextual_seq_len, + max_attn_len, + LOCK, + off_h, + stride_qh, + stride_doh, + stride_qm, + stride_dom, + stride_dqm, + alpha, + MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_M: tl.constexpr, + ATOMIC_ADD: tl.constexpr, + ENABLE_TMA: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, +): + pos_offs_m = offs_m + start_m + mask_m = pos_offs_m < seq_len + invalid_mask_trans = pos_offs_m[None, :] == offs_n[:, None] + # recompute qk and silu + if HAS_CONTEXTUAL_SEQ_LEN: + pos_offs_m = pos_offs_m - contextual_seq_len + 1 + pos_offs_m = tl.where( + pos_offs_m > 0, + pos_offs_m, + 0, + ) + if HAS_MULTIPLE_TARGETS: + pos_offs_m = tl.where( + pos_offs_m < max_ids, + pos_offs_m, + max_ids, + ) + if ENABLE_TMA: + q = device_desc_q.load( + [start_m, (off_h * stride_qh).to(tl.int32)], + ) + q_trans = tl.trans(q) + else: + q_trans = tl.load( + q_ptrs_trans + start_m * stride_qm, + mask=mask_m[None, :], + other=0.0, + ) + qk_trans = tl.dot(k, q_trans, allow_tf32=ALLOW_TF32) * alpha + sig_trans = fast_dividef(1.0, 1.0 + tl.exp(-qk_trans)) + silu_trans = qk_trans * sig_trans * (1.0 / MAX_SEQ_LEN) + pos_offs_m_minus_n = pos_offs_m[None, :] - pos_offs_n[:, None] + invalid_mask_trans = invalid_mask_trans or (pos_offs_m_minus_n > 0) + if HAS_MAX_ATTN_LEN: + invalid_mask_trans = invalid_mask_trans and pos_offs_m_minus_n <= max_attn_len + if HAS_CONTEXTUAL_SEQ_LEN: + invalid_mask_trans = invalid_mask_trans or ( + pos_offs_m[None, :] == 0 and pos_offs_n[:, None] < max_ids + ) + silu_trans = tl.where(invalid_mask_trans, silu_trans, 0) + silu_trans = silu_trans.to(k.dtype) + # compute dv + if ENABLE_TMA: + do = device_desc_do.load( + [start_m, (off_h * stride_doh).to(tl.int32)], + ) + else: + do = tl.load( + do_ptrs + start_m * stride_dom, + mask=mask_m[:, None], + other=0.0, + ) + dv += tl.dot(silu_trans, do, allow_tf32=ALLOW_TF32) + + # compute dk and dq + dqk_trans = tl.dot(v, tl.trans(do), allow_tf32=ALLOW_TF32) + dqk_trans = ( + dqk_trans * sig_trans * (1 + qk_trans * (1 - sig_trans)) * (1.0 / MAX_SEQ_LEN) + ) + dqk_trans = tl.where(invalid_mask_trans, dqk_trans, 0) + dqk_trans = dqk_trans.to(k.dtype) + + # Note: the factor `alpha` is delayed until the end of the function to reduce the cost + dk += tl.dot(dqk_trans, tl.trans(q_trans), allow_tf32=ALLOW_TF32) + acc_dq( + dq_ptrs_trans=dq_ptrs_trans, + start_m=start_m, + stride_dqm=stride_dqm, + k=k, + dqk_trans=dqk_trans, + alpha=alpha, + mask_m=mask_m, + MAX_SEQ_LEN=MAX_SEQ_LEN, + LOCK=LOCK, + BLOCK_M=BLOCK_M, + ATOMIC_ADD=ATOMIC_ADD, + ALLOW_TF32=ALLOW_TF32, + ) + return dk, dv + + +@triton.jit +def _hstu_attn_bwd_one_col_block( # noqa C901 + start_n, + seq_len, + n_targets, + contextual_seq_len, + max_attn_len, + Q, + K, + V, + DOut, + DQ, + DK, + DV, + device_desc_q, + device_desc_k, + device_desc_v, + device_desc_do, + device_desc_dk, + device_desc_dv, + LOCK, + off_h, + stride_qh, + stride_kh, + stride_vh, + stride_doh, + stride_dkh, + stride_dvh, + stride_qm, + stride_kn, + stride_vn, + stride_dom, + stride_dqm, + stride_dkn, + stride_dvn, + alpha, + MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + UNROLL: tl.constexpr, + ATOMIC_ADD: tl.constexpr, + ENABLE_TMA: tl.constexpr, +): + if HAS_MULTIPLE_TARGETS: + low = start_n + if HAS_MAX_ATTN_LEN: + high = start_n + max_attn_len + BLOCK_N + high = high if high + n_targets < seq_len else seq_len + else: + high = seq_len + else: + low = start_n + if HAS_MAX_ATTN_LEN: + high = start_n + max_attn_len + BLOCK_N + high = high if high < seq_len else seq_len + else: + high = seq_len + if HAS_CONTEXTUAL_SEQ_LEN: + contextual_block_end = tl.cdiv(contextual_seq_len, BLOCK_M) * BLOCK_M + if low < contextual_block_end: + low = contextual_block_end + + offs_m = tl.arange(0, BLOCK_M) + offs_qk_d = tl.arange(0, BLOCK_D_Q) + offs_v_d = tl.arange(0, BLOCK_D_V) + offs_n = start_n + tl.arange(0, BLOCK_N) + + dq_ptrs_trans = DQ + (offs_m[None, :] * stride_dqm + offs_qk_d[:, None]) + dv = tl.zeros([BLOCK_N, BLOCK_D_V], dtype=tl.float32) + dk = tl.zeros([BLOCK_N, BLOCK_D_Q], dtype=tl.float32) + if ENABLE_TMA: + q_ptrs_trans = None + do_ptrs = None + k = device_desc_k.load( + [start_n, (off_h * stride_kh).to(tl.int32)], + ) + v = device_desc_v.load( + [start_n, (off_h * stride_vh).to(tl.int32)], + ) + else: + mask_n = offs_n < seq_len + q_ptrs_trans = Q + (offs_m[None, :] * stride_qm + offs_qk_d[:, None]) + do_ptrs = DOut + (offs_m[:, None] * stride_dom + offs_v_d[None, :]) + k_ptrs = K + (offs_n[:, None] * stride_kn + offs_qk_d[None, :]) + v_ptrs = V + (offs_n[:, None] * stride_vn + offs_v_d[None, :]) + k = tl.load(k_ptrs, mask=mask_n[:, None], other=0.0) + v = tl.load(v_ptrs, mask=mask_n[:, None], other=0.0) + max_ids = seq_len + if HAS_CONTEXTUAL_SEQ_LEN: + pos_offs_n = offs_n - contextual_seq_len + 1 + pos_offs_n = tl.where( + pos_offs_n > 0, + pos_offs_n, + 0, + ) + max_ids = max_ids - contextual_seq_len + 1 + else: + pos_offs_n = offs_n + if HAS_MULTIPLE_TARGETS: + max_ids = max_ids - n_targets + pos_offs_n = tl.where( + pos_offs_n < max_ids, + pos_offs_n, + max_ids, + ) + # loop over rows + if HAS_CONTEXTUAL_SEQ_LEN: + for start_m in range(0, contextual_seq_len, BLOCK_M): + start_m = tl.multiple_of(start_m, BLOCK_M) + dk, dv = _hstu_attn_bwd_one_block( + start_m=start_m, + offs_n=offs_n, + offs_m=offs_m, + q_ptrs_trans=q_ptrs_trans, + dq_ptrs_trans=dq_ptrs_trans, + do_ptrs=do_ptrs, + device_desc_q=device_desc_q, + device_desc_do=device_desc_do, + dk=dk, + dv=dv, + k=k, + v=v, + pos_offs_n=pos_offs_n, + seq_len=seq_len, + max_ids=max_ids, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + LOCK=LOCK, + off_h=off_h, + stride_qh=stride_qh, + stride_doh=stride_doh, + stride_qm=stride_qm, + stride_dom=stride_dom, + stride_dqm=stride_dqm, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_M=BLOCK_M, + ATOMIC_ADD=ATOMIC_ADD, + ENABLE_TMA=ENABLE_TMA, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + ) + for start_m in tl.range(low, high, BLOCK_M, loop_unroll_factor=UNROLL): + start_m = tl.multiple_of(start_m, BLOCK_M) + dk, dv = _hstu_attn_bwd_one_block( + start_m=start_m, + offs_n=offs_n, + offs_m=offs_m, + q_ptrs_trans=q_ptrs_trans, + dq_ptrs_trans=dq_ptrs_trans, + do_ptrs=do_ptrs, + device_desc_q=device_desc_q, + device_desc_do=device_desc_do, + dk=dk, + dv=dv, + k=k, + v=v, + pos_offs_n=pos_offs_n, + seq_len=seq_len, + max_ids=max_ids, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + LOCK=LOCK, + off_h=off_h, + stride_qh=stride_qh, + stride_doh=stride_doh, + stride_qm=stride_qm, + stride_dom=stride_dom, + stride_dqm=stride_dqm, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_M=BLOCK_M, + ATOMIC_ADD=ATOMIC_ADD, + ENABLE_TMA=ENABLE_TMA, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + ) + # write-back + dk = dk * alpha + if ENABLE_TMA: + device_desc_dv.store( + [start_n, (off_h * stride_dvh).to(tl.int32)], + dv.to(k.dtype), + ) + device_desc_dk.store( + [start_n, (off_h * stride_dkh).to(tl.int32)], + dk.to(k.dtype), + ) + else: + dv_ptrs = DV + (offs_n[:, None] * stride_dvn + offs_v_d[None, :]) + dk_ptrs = DK + (offs_n[:, None] * stride_dkn + offs_qk_d[None, :]) + tl.store(dv_ptrs, dv.to(k.dtype), mask=mask_n[:, None]) # pyre-ignore[61] + tl.store(dk_ptrs, dk.to(k.dtype), mask=mask_n[:, None]) # pyre-ignore[61] + + +def _bwd_pre_hook(nargs): + nargs["DQ"].zero_() + if nargs["SEQUENCE_PARALLEL"] is True: + nargs["LOCK"].zero_() + + +def _get_bw_configs() -> List[triton.Config]: + if torch.version.hip: + configs = [] + for BLOCK_M in [32, 64]: + for BLOCK_N in [32, 64, 128]: + for num_stages in [1, 2]: + for num_warps in [4, 8]: + for matrix_instr_nonkdim in [16, 32]: + for waves_per_eu in [0, 2, 4]: + for sp in [True, False]: + configs.append( + triton.Config( + { + "BLOCK_M": BLOCK_M, + "BLOCK_N": BLOCK_N, + "matrix_instr_nonkdim": matrix_instr_nonkdim, + "waves_per_eu": waves_per_eu, + "SEQUENCE_PARALLEL": sp, + "UNROLL": 1, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=_bwd_pre_hook, + ) + ) + return configs + + configs = [ + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=2, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 16, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=2, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=3, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128, "SEQUENCE_PARALLEL": False, "UNROLL": 4}, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 32, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=2, + num_warps=2, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 32, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + ] + if torch.cuda.is_available() and torch.version.cuda < "12.8": + configs += [ + triton.Config( + {"BLOCK_M": 16, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": False, "UNROLL": 1}, + num_stages=1, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=1, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 128, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=3, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "SEQUENCE_PARALLEL": True, "UNROLL": 1}, + num_stages=2, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + triton.Config( + { + "BLOCK_M": 32, + "BLOCK_N": 128, + "SEQUENCE_PARALLEL": False, + "UNROLL": 2, + }, + num_stages=2, + num_warps=8, + pre_hook=_bwd_pre_hook, + ), + ] + else: + print("WARNING: temporarily disabled some autotune configs for CUDA 12.8+") + return configs + + +def _get_bw_pinned_configs() -> List[triton.Config]: + # Pinned backward-attention configs for MI350X gfx950. Pins the fast + # matrix_instr_nonkdim=16 winner(s) to avoid the 16-vs-32 autotune lottery. + # + # LIST, one entry per tuned shape (currently just bs=1024). With >1 entry the + # autotuner benchmarks only these candidates and caches the winner per `key` + # (AUTOTUNE_Z / H / AUTOTUNE_MAX_SEQ_LEN / DimQ / DimV), so each batch size + # picks its own config automatically — same pattern as the layer-norm pins. + # + # TO ADD A NEW BATCH SIZE / SHAPE: + # 1. Run once with TRITON_FULL_AUTOTUNE=1 TRITON_PRINT_AUTOTUNING=1. + # 2. Grep the log for "best config selected:" under "_hstu_attn_bwd". + # 3. Append that config below (verbatim BLOCK_M/BLOCK_N/matrix_instr_nonkdim/ + # waves_per_eu/SEQUENCE_PARALLEL/UNROLL/num_stages/num_warps). + # Keep pre_hook=_bwd_pre_hook on every entry (the bwd configs require it). + if torch.version.hip: + return [ + # --- yambda bs=1024, L=2048 winner (from capture log) --- + triton.Config( + { + "BLOCK_M": 32, + "BLOCK_N": 128, + "matrix_instr_nonkdim": 16, + "waves_per_eu": 0, + "SEQUENCE_PARALLEL": False, + "UNROLL": 1, + }, + num_stages=1, + num_warps=4, + pre_hook=_bwd_pre_hook, + ), + # --- add more (bs, L) winners here; see "TO ADD A NEW BATCH SIZE" --- + ] + return _get_bw_configs() + + +@triton_autotune( + configs=pinned_or_full(_get_bw_pinned_configs(), _get_bw_configs), + key=[ + "AUTOTUNE_Z", + "H", + "AUTOTUNE_MAX_SEQ_LEN", + "DimQ", + "DimV", + ], +) +@triton.jit +def _hstu_attn_bwd( # noqa C901 + Q, + K, + V, + tma_workspace_ptr, + sort_by_length_indices, + seq_offsets, + num_targets, + DOut, + DQ, + DK, + DV, + LOCK, + stride_qm, + stride_qh, + stride_kn, + stride_kh, + stride_vn, + stride_vh, + stride_dom, + stride_doh, + stride_dqm, + stride_dqh, + stride_dkn, + stride_dkh, + stride_dvn, + stride_dvh, + alpha, + contextual_seq_len, + max_attn_len, + Z, + AUTOTUNE_Z, + H, + MAX_SEQ_LEN, + AUTOTUNE_MAX_SEQ_LEN, # Quantized MAX_SEQ_LEN used as an autotuning key + DimQ, + DimV, + HAS_MULTIPLE_TARGETS: tl.constexpr, + HAS_CONTEXTUAL_SEQ_LEN: tl.constexpr, + HAS_MAX_ATTN_LEN: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_D_Q: tl.constexpr, + BLOCK_D_V: tl.constexpr, + SEQUENCE_PARALLEL: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + UNROLL: tl.constexpr, + HAS_SORT_BY_LENGTH_INDICES: tl.constexpr, + ENABLE_TMA: tl.constexpr, + TMA_DESC_SIZE: tl.constexpr, + ENABLE_BUFFER_OPS_ASSUMES: tl.constexpr, +): + off_hz = tl.program_id(0) + off_z = off_hz // H + if HAS_SORT_BY_LENGTH_INDICES: + off_z = tl.load(sort_by_length_indices + off_z) + off_h = off_hz % H + off_h = off_h.to(tl.int64) + seq_start = tl.load(seq_offsets + off_z).to(tl.int64) + seq_end = tl.load(seq_offsets + off_z + 1) + seq_len = (seq_end - seq_start).to(tl.int32) + if HAS_MULTIPLE_TARGETS: + n_targets = tl.load(num_targets + off_z).to(tl.int32) + else: + n_targets = None + if ENABLE_BUFFER_OPS_ASSUMES: + tl.assume(off_hz >= 0) + tl.assume(off_z >= 0) + tl.assume(off_h >= 0) + tl.assume(seq_start >= 0) + tl.assume(stride_qm >= 0) + tl.assume(stride_qh >= 0) + tl.assume(stride_kn >= 0) + tl.assume(stride_kh >= 0) + tl.assume(stride_vn >= 0) + tl.assume(stride_vh >= 0) + tl.assume(stride_dom >= 0) + tl.assume(stride_doh >= 0) + tl.assume(stride_dqm >= 0) + tl.assume(stride_dqh >= 0) + tl.assume(stride_dkn >= 0) + tl.assume(stride_dkh >= 0) + tl.assume(stride_dvn >= 0) + tl.assume(stride_dvh >= 0) + + # offset pointers for batch/head + Q = Q + seq_start * stride_qm + K = K + seq_start * stride_kn + V = V + seq_start * stride_vn + DOut = DOut + seq_start * stride_dom + DQ = DQ + seq_start * stride_dqm + off_h * stride_dqh + DK = DK + seq_start * stride_dkn + DV = DV + seq_start * stride_dvn + device_desc_q = None + device_desc_k = None + device_desc_v = None + device_desc_do = None + device_desc_dk = None + device_desc_dv = None + if ENABLE_TMA: + device_desc_q = tl.make_tensor_descriptor( + Q, + shape=[seq_len, H * DimQ], + strides=[H * DimQ, 1], + block_shape=[BLOCK_M, BLOCK_D_Q], + ) + device_desc_do = tl.make_tensor_descriptor( + DOut, + shape=[seq_len, H * DimV], + strides=[H * DimV, 1], + block_shape=[BLOCK_M, BLOCK_D_V], + ) + device_desc_k = tl.make_tensor_descriptor( + K, + shape=[seq_len, H * DimQ], + strides=[H * DimQ, 1], + block_shape=[BLOCK_N, BLOCK_D_Q], + ) + device_desc_dk = tl.make_tensor_descriptor( + DK, + shape=[seq_len, H * DimQ], + strides=[H * DimQ, 1], + block_shape=[BLOCK_N, BLOCK_D_Q], + ) + device_desc_v = tl.make_tensor_descriptor( + V, + shape=[seq_len, H * DimV], + strides=[H * DimV, 1], + block_shape=[BLOCK_N, BLOCK_D_V], + ) + device_desc_dv = tl.make_tensor_descriptor( + DV, + shape=[seq_len, H * DimV], + strides=[H * DimV, 1], + block_shape=[BLOCK_N, BLOCK_D_V], + ) + else: + Q += off_h * stride_qh + K += off_h * stride_kh + V += off_h * stride_vh + DOut += off_h * stride_doh + DK += off_h * stride_dkh + DV += off_h * stride_dvh + if SEQUENCE_PARALLEL: + start_n = tl.program_id(1) * BLOCK_N + if start_n >= seq_len: + return + _hstu_attn_bwd_one_col_block( + start_n=start_n, + seq_len=seq_len, + n_targets=n_targets, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Q=Q, + K=K, + V=V, + DOut=DOut, + DQ=DQ, + DK=DK, + DV=DV, + device_desc_q=device_desc_q, + device_desc_k=device_desc_k, + device_desc_v=device_desc_v, + device_desc_do=device_desc_do, + device_desc_dk=device_desc_dk, + device_desc_dv=device_desc_dv, + LOCK=LOCK, + off_h=off_h, + stride_qh=stride_qh, + stride_kh=stride_kh, + stride_vh=stride_vh, + stride_doh=stride_doh, + stride_dkh=stride_dkh, + stride_dvh=stride_dvh, + stride_qm=stride_qm, + stride_kn=stride_kn, + stride_vn=stride_vn, + stride_dom=stride_dom, + stride_dqm=stride_dqm, + stride_dkn=stride_dkn, + stride_dvn=stride_dvn, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + UNROLL=UNROLL, + ATOMIC_ADD=True, + ENABLE_TMA=ENABLE_TMA, + ) + else: + for start_n in range(0, seq_len, BLOCK_N): + _hstu_attn_bwd_one_col_block( + start_n=start_n, + seq_len=seq_len, + n_targets=n_targets, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Q=Q, + K=K, + V=V, + DOut=DOut, + DQ=DQ, + DK=DK, + DV=DV, + device_desc_q=device_desc_q, + device_desc_k=device_desc_k, + device_desc_v=device_desc_v, + device_desc_do=device_desc_do, + device_desc_dk=device_desc_dk, + device_desc_dv=device_desc_dv, + LOCK=LOCK, + off_h=off_h, + stride_qh=stride_qh, + stride_kh=stride_kh, + stride_vh=stride_vh, + stride_doh=stride_doh, + stride_dkh=stride_dkh, + stride_dvh=stride_dvh, + stride_qm=stride_qm, + stride_kn=stride_kn, + stride_vn=stride_vn, + stride_dom=stride_dom, + stride_dqm=stride_dqm, + stride_dkn=stride_dkn, + stride_dvn=stride_dvn, + alpha=alpha, + MAX_SEQ_LEN=MAX_SEQ_LEN, + HAS_MULTIPLE_TARGETS=HAS_MULTIPLE_TARGETS, + HAS_CONTEXTUAL_SEQ_LEN=HAS_CONTEXTUAL_SEQ_LEN, + HAS_MAX_ATTN_LEN=HAS_MAX_ATTN_LEN, + ALLOW_TF32=ALLOW_TF32, + BLOCK_D_Q=BLOCK_D_Q, + BLOCK_D_V=BLOCK_D_V, + BLOCK_M=BLOCK_M, + BLOCK_N=BLOCK_N, + UNROLL=UNROLL, + ATOMIC_ADD=False, + ENABLE_TMA=ENABLE_TMA, + ) + + +@maybe_register_custom_op( + "generative_recommenders::triton_hstu_attention_fwd", mutates_args=() +) +def triton_hstu_attention_fwd( + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + sort_by_length_indices: Optional[torch.Tensor], + enable_tma: bool, + num_softmax_heads: int, +) -> torch.Tensor: + Z = seq_offsets.numel() - 1 + AUTOTUNE_Z = prev_power_of_2(Z) + L, H, DimQ = q.shape + _, _, DimV = v.shape + out = torch.empty_like(v) + has_multiple_targets = num_targets is not None + has_contextual_seq_len = contextual_seq_len > 0 + has_max_attn_len = max_attn_len > 0 + has_sort_by_length_indices = sort_by_length_indices is not None + if L == 0: + return out + + TMA_DESC_SIZE = 128 + workspace = None + desc_q = q + desc_k = k + desc_v = v + + if enable_tma and tensor_descriptor_tma: + dummy_block = [1, 1] + desc_q = TensorDescriptor( + q, + shape=[L, H * DimQ], + strides=[H * DimQ, 1], + block_shape=dummy_block, + ) + desc_v = TensorDescriptor( + v, + shape=[L, H * DimV], + strides=[H * DimV, 1], + block_shape=dummy_block, + ) + desc_k = TensorDescriptor( + k, + shape=[L, H * DimQ], + strides=[H * DimQ, 1], + block_shape=dummy_block, + ) + + def alloc_fn(size: int, align: int, stream: Optional[int]): + assert align == TMA_DESC_SIZE + return torch.empty(size, dtype=torch.int8, device="cuda") + + # pyre-ignore [6] + triton.set_allocator(alloc_fn) + grid = lambda meta: ( # noqa E731 + triton.cdiv(N, meta["BLOCK_M"]), + Z * H, + ) + + _hstu_attn_fwd[grid]( + Q=desc_q, + K=desc_k, + V=desc_v, + workspace_ptr=workspace, + sort_by_length_indices=sort_by_length_indices, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=out, + stride_qm=q.stride(0), + stride_qh=q.stride(1), + stride_kn=k.stride(0), + stride_kh=k.stride(1), + stride_vn=v.stride(0), + stride_vh=v.stride(1), + stride_om=out.stride(0), + stride_oh=out.stride(1), + alpha=alpha, + Z=Z, + AUTOTUNE_Z=AUTOTUNE_Z, + H=H, + MAX_SEQ_LEN=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), + DimQ=DimQ, + DimV=DimV, + DeltaSize=0, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + HAS_MULTIPLE_TARGETS=has_multiple_targets, + IS_DELTA_Q=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BLOCK_D_Q=DimQ, + BLOCK_D_V=DimV, + HAS_CONTEXTUAL_SEQ_LEN=has_contextual_seq_len, + HAS_MAX_ATTN_LEN=has_max_attn_len, + HAS_SORT_BY_LENGTH_INDICES=has_sort_by_length_indices, + ENABLE_TMA=enable_tma, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ) + return out + + +@maybe_register_custom_op( + "generative_recommenders::triton_hstu_attention_bwd", + mutates_args=("dq", "dk", "dv"), +) +def triton_hstu_attention_bwd( + dout: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + dq: torch.Tensor, + dk: torch.Tensor, + dv: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + N: int, + alpha: float, + max_attn_len: int, + contextual_seq_len: int, + sort_by_length_indices: Optional[torch.Tensor], + enable_tma: bool, + num_softmax_heads: int, +) -> None: + orig_dq, orig_dk, orig_dv = dq, dk, dv + dout = switch_to_contiguous_if_needed(dout) + dq = switch_to_contiguous_if_needed(dq) + dk = switch_to_contiguous_if_needed(dk) + dv = switch_to_contiguous_if_needed(dv) + if dout.shape[0] == 0: + orig_dq.zero_() + orig_dk.zero_() + orig_dv.zero_() + return + Z = seq_offsets.numel() - 1 + _, H, DimQ = q.shape + _, _, DimV = v.shape + grid = lambda meta: ( # noqa E731 + Z * H, + (triton.cdiv(N, meta["BLOCK_N"]) if meta["SEQUENCE_PARALLEL"] else 1), + ) + # The minimum size of BLOCK_M used in `_get_bw_configs`. + # TODO (linjianma): avoid hardcoding the value. + MIN_BLOCK_M = 16 + lock = torch.empty( + (Z * H, triton.cdiv(N, MIN_BLOCK_M)), + dtype=torch.int32, + device=q.device, + ) + AUTOTUNE_Z = prev_power_of_2(Z) + TMA_DESC_SIZE = 128 + tma_workspace = None + + def alloc_fn(size: int, align: int, stream: Optional[int]): + assert align == TMA_DESC_SIZE + return torch.empty(size, dtype=torch.int8, device="cuda") + + # pyre-ignore [6] + triton.set_allocator(alloc_fn) + + # Enable BufferOps on AMD + ENABLE_BUFFER_OPS_ASSUMES = torch.version.hip is not None + _hstu_attn_bwd[grid]( + Q=q, + K=k, + V=v, + tma_workspace_ptr=tma_workspace, + sort_by_length_indices=sort_by_length_indices, + seq_offsets=seq_offsets, + num_targets=num_targets, + DOut=dout, + DQ=dq, + DK=dk, + DV=dv, + LOCK=lock, + stride_qm=q.stride(0), + stride_qh=q.stride(1), + stride_kn=k.stride(0), + stride_kh=k.stride(1), + stride_vn=v.stride(0), + stride_vh=v.stride(1), + stride_dom=dout.stride(0), + stride_doh=dout.stride(1), + stride_dqm=dq.stride(0), + stride_dqh=dq.stride(1), + stride_dkn=dk.stride(0), + stride_dkh=dk.stride(1), + stride_dvn=dv.stride(0), + stride_dvh=dv.stride(1), + alpha=alpha, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Z=Z, + AUTOTUNE_Z=AUTOTUNE_Z, + H=H, + MAX_SEQ_LEN=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), + DimQ=DimQ, + DimV=DimV, + HAS_MULTIPLE_TARGETS=num_targets is not None, + HAS_CONTEXTUAL_SEQ_LEN=contextual_seq_len > 0, + HAS_MAX_ATTN_LEN=max_attn_len > 0, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BLOCK_D_Q=DimQ, + BLOCK_D_V=DimV, + HAS_SORT_BY_LENGTH_INDICES=sort_by_length_indices is not None, + ENABLE_TMA=enable_tma, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ENABLE_BUFFER_OPS_ASSUMES=ENABLE_BUFFER_OPS_ASSUMES, + ) + + copy_if_different_ptr(orig_dq, dq) + copy_if_different_ptr(orig_dk, dk) + copy_if_different_ptr(orig_dv, dv) + + +@triton_hstu_attention_fwd.register_fake +def _triton_hstu_attention_fwd_fake( + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + sort_by_length_indices: Optional[torch.Tensor], + enable_tma: bool, + num_softmax_heads: int, +) -> torch.Tensor: + L, H, _ = q.shape + _, _, DimV = v.shape + out = torch.empty((L, H, DimV), dtype=v.dtype, device=v.device) + return out + + +@triton_hstu_attention_bwd.register_fake +def _triton_hstu_attention_bwd_fake( + dout: torch.Tensor, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + dq: torch.Tensor, + dk: torch.Tensor, + dv: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + N: int, + alpha: float, + max_attn_len: int, + contextual_seq_len: int, + sort_by_length_indices: Optional[torch.Tensor], + enable_tma: bool, + num_softmax_heads: int, +) -> None: + return None + + +class _AttentionFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + sort_by_length: bool, + enable_tma: bool, + ) -> torch.Tensor: + sort_by_length_indices = None + if sort_by_length: + seq_lengths = seq_offsets[1:] - seq_offsets[:-1] + _, sort_by_length_indices = torch.sort( + seq_lengths, descending=True, stable=False + ) + saved_tensors = [q, k, v, seq_offsets] + if num_targets is not None: + saved_tensors.append(num_targets) + if sort_by_length_indices is not None: + saved_tensors.append(sort_by_length_indices) + ctx.save_for_backward(*saved_tensors) + ctx.alpha = alpha + ctx.has_multiple_targets = num_targets is not None + ctx.max_attn_len = max_attn_len + ctx.N = N + ctx.contextual_seq_len = contextual_seq_len + ctx.sort_by_length = sort_by_length + ctx.enable_tma = enable_tma + return triton_hstu_attention_fwd( + N=N, + alpha=alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length_indices=sort_by_length_indices, + enable_tma=enable_tma, + num_softmax_heads=0, + ) + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dout: torch.Tensor + ) -> Tuple[ + None, + None, + torch.Tensor, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + ]: + with torch.inference_mode(): + q, k, v, seq_offsets = ctx.saved_tensors[:4] + idx = 4 + if ctx.has_multiple_targets: + num_targets = ctx.saved_tensors[idx] + idx += 1 + else: + num_targets = None + if ctx.sort_by_length: + sort_by_length_indices = ctx.saved_tensors[idx] + else: + sort_by_length_indices = None + + dq = torch.empty_like(q) + dk = torch.empty_like(k) + dv = torch.empty_like(v) + triton_hstu_attention_bwd( + dout=dout, + q=q, + k=k, + v=v, + dq=dq, + dk=dk, + dv=dv, + seq_offsets=seq_offsets, + num_targets=num_targets, + N=ctx.N, + alpha=ctx.alpha, + max_attn_len=ctx.max_attn_len, + contextual_seq_len=ctx.contextual_seq_len, + sort_by_length_indices=sort_by_length_indices, + enable_tma=ctx.enable_tma, + num_softmax_heads=0, + ) + return ( + None, + None, + dq, + dk, + dv, + None, + None, + None, + None, + None, + None, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_hstu_mha( + N: int, + alpha: float, + q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + sort_by_length: bool = False, + enable_tma: bool = False, +) -> torch.Tensor: + return _AttentionFunction.apply( + N, + alpha, + q, + k, + v, + seq_offsets, + num_targets, + max_attn_len, + contextual_seq_len, + sort_by_length, + enable_tma, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_cached_hstu_mha( + N: int, + alpha: float, + delta_q: torch.Tensor, + k: torch.Tensor, + v: torch.Tensor, + seq_offsets: torch.Tensor, + num_targets: Optional[torch.Tensor] = None, + max_attn_len: int = 0, + contextual_seq_len: int = 0, + enable_tma: bool = False, +) -> torch.Tensor: + Z = seq_offsets.size(0) - 1 + AUTOTUNE_Z = prev_power_of_2(Z) + DELTA_L, H, DimQ = delta_q.shape + DeltaSize = DELTA_L // Z + L, _, DimV = v.shape + out = torch.empty((DELTA_L, H, DimV), dtype=delta_q.dtype, device=delta_q.device) + + TMA_DESC_SIZE = 128 + desc_q = delta_q + desc_k = k + desc_v = v + + if enable_tma and tensor_descriptor_tma: + dummy_block = [1, 1] + desc_q = TensorDescriptor( + delta_q, + shape=[DELTA_L, H * DimQ], + strides=[H * DimQ, 1], + block_shape=dummy_block, + ) + desc_v = TensorDescriptor( + v, + shape=[L, H * DimV], + strides=[H * DimV, 1], + block_shape=dummy_block, + ) + desc_k = TensorDescriptor( + k, + shape=[L, H * DimQ], + strides=[H * DimQ, 1], + block_shape=dummy_block, + ) + + def alloc_fn(size: int, align: int, stream: Optional[int]): + assert align == TMA_DESC_SIZE + return torch.empty(size, dtype=torch.int8, device="cuda") + + # pyre-ignore [6] + triton.set_allocator(alloc_fn) + grid = lambda meta: ( # noqa E731 + triton.cdiv(DeltaSize, meta["BLOCK_M"]), + Z * H, + ) + + has_contextual_seq_len = contextual_seq_len > 0 + has_max_attn_len = max_attn_len > 0 + _hstu_attn_fwd[grid]( + Q=desc_q, + K=desc_k, + V=desc_v, + workspace_ptr=None, + sort_by_length_indices=None, + seq_offsets=seq_offsets, + num_targets=num_targets, + Out=out, + stride_qm=delta_q.stride(0), + stride_qh=delta_q.stride(1), + stride_kn=k.stride(0), + stride_kh=k.stride(1), + stride_vn=v.stride(0), + stride_vh=v.stride(1), + stride_om=out.stride(0), + stride_oh=out.stride(1), + alpha=alpha, + contextual_seq_len=contextual_seq_len, + max_attn_len=max_attn_len, + Z=Z, + AUTOTUNE_Z=AUTOTUNE_Z, + H=H, + MAX_SEQ_LEN=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(N), + DimQ=DimQ, + DimV=DimV, + DeltaSize=DeltaSize, + HAS_MULTIPLE_TARGETS=num_targets is not None, + IS_DELTA_Q=True, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + BLOCK_D_Q=DimQ, + BLOCK_D_V=DimV, + HAS_CONTEXTUAL_SEQ_LEN=has_contextual_seq_len, + HAS_MAX_ATTN_LEN=has_max_attn_len, + HAS_SORT_BY_LENGTH_INDICES=False, + ENABLE_TMA=enable_tma, + TMA_DESC_SIZE=TMA_DESC_SIZE, + ) + return out diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py new file mode 100644 index 000000000..516a15664 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_linear.py @@ -0,0 +1,3047 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.triton.triton_addmm import maybe_triton_addmm_fwd +from generative_recommenders.ops.utils import maybe_register_custom_op + + +def _get_layer_norm_mul_dropout_fwd_multirow_configs() -> List[triton.Config]: + """Generate autotune configs for multi-row LayerNorm multiplication with dropout kernels.""" + configs = [] + for BLOCK_N in [1, 2, 4, 8, 16]: + for num_warps in [1, 2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +from generative_recommenders.ops.utils import use_separated_rng_ln_mul_dropout + +# @manual=//triton:triton +from triton.language.extra import libdevice + +try: + # @manual=//triton:triton + from triton.language.extra.libdevice import fast_dividef +except ImportError: + try: + # @manual=//triton:triton + from triton.language.extra.cuda.libdevice import fast_dividef + except ImportError: + # pyre-ignore: Undefined import [21] + # @manual=//triton:triton + from triton.language.math import fast_dividef + + +COMPUTE_OUTPUT_LN_FAST_DROPOUT = False + + +def set_compute_output_ln_fast_dropout(value: bool) -> None: + global COMPUTE_OUTPUT_LN_FAST_DROPOUT + COMPUTE_OUTPUT_LN_FAST_DROPOUT = value + + +FUSE_OUTPUT_LN_RNG_BLACKWELL = False + + +# Only impact B200 training when CONCAT_UX is False +def set_fuse_output_ln_rng_blackwell(value: bool) -> None: + global FUSE_OUTPUT_LN_RNG_BLACKWELL + FUSE_OUTPUT_LN_RNG_BLACKWELL = value + + +@triton.jit +def rand3x(seed, offsets, n_rounds: tl.constexpr = 10): # pyre-ignore [9] + i1, i2, i3, _ = tl.randint4x(seed, offsets, n_rounds) + u1 = tl.uint_to_uniform_float(i1) + u2 = tl.uint_to_uniform_float(i2) + u3 = tl.uint_to_uniform_float(i3) + return u1, u2, u3 + + +@triton.jit +def _generate_random_mask( + MASK_BUFFER, + N, + dropout_ratio, + seed, + D: tl.constexpr, + STRIDE: tl.constexpr, + BLOCK_D: tl.constexpr, + NUM_MASKS: tl.constexpr, +): + """Generate bit-packed dropout masks for (N, D) tensors. Outputs int8. + + Processes 4 rows per program using rand4x. Mask j occupies bit j. + Extraction: y = val & 1, x = val & 2, u = val & 4. + """ + pid = tl.program_id(0) + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + start_row = pid.to(tl.int64) * 4 + + base_ptr = MASK_BUFFER + start_row * STRIDE + cols + row0_mask = (start_row < N) & col_mask + row1_mask = ((start_row + 1) < N) & col_mask + row2_mask = ((start_row + 2) < N) & col_mask + row3_mask = ((start_row + 3) < N) & col_mask + + # Each pid uses NUM_MASKS consecutive BLOCK_D chunks for Philox offsets + rand_offset = pid * (NUM_MASKS * BLOCK_D) + cols + + packed0 = tl.zeros([BLOCK_D], dtype=tl.int8) + packed1 = tl.zeros([BLOCK_D], dtype=tl.int8) + packed2 = tl.zeros([BLOCK_D], dtype=tl.int8) + packed3 = tl.zeros([BLOCK_D], dtype=tl.int8) + + for j in tl.static_range(NUM_MASKS): + r0, r1, r2, r3 = tl.rand4x(seed, rand_offset) + packed0 |= (r0 > dropout_ratio).to(tl.int8) << j + packed1 |= (r1 > dropout_ratio).to(tl.int8) << j + packed2 |= (r2 > dropout_ratio).to(tl.int8) << j + packed3 |= (r3 > dropout_ratio).to(tl.int8) << j + rand_offset += BLOCK_D + + tl.store(base_ptr, packed0, mask=row0_mask) + tl.store(base_ptr + STRIDE, packed1, mask=row1_mask) + tl.store(base_ptr + 2 * STRIDE, packed2, mask=row2_mask) + tl.store(base_ptr + 3 * STRIDE, packed3, mask=row3_mask) + + +@triton_autotune( + configs=_get_layer_norm_mul_dropout_fwd_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def _ln_mul_dropout_fwd_rng( + X, + U, + Y, + W, + B, + Mean, + Rstd, + RANDOM_MASK, + N, + D, + eps, + dropout_ratio, + stride_x, + stride_u, + stride_y, + stride_mask, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_U: tl.constexpr, + CONCAT_X: tl.constexpr, + MUL_U_ACTIVATION_TYPE: tl.constexpr, +): + block_id = tl.program_id(0) + start_row = block_id * BLOCK_N + + # Create block pointers for X, U, and Y + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + U_block_ptr = tl.make_block_ptr( + base=U, + shape=(N, D), + strides=(stride_u, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + # Load data blocks + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + u_block = tl.load(U_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + # Pre-compute 2D mask for reuse in dropout and masked operations + mask_2d = row_mask[:, None] & col_mask[None, :] + + # Pre-compute inv_D to replace divisions with multiplications (optimization) + inv_D = 1.0 / D + + mean = tl.sum(x_block, axis=1) * inv_D + tl.store(Mean + rows, mean, mask=row_mask) + mean = tl.expand_dims(mean, 1) + + x_mean = x_block - mean + x_mean = tl.where(mask_2d, x_mean, 0.0) + _var = x_mean * x_mean + var = tl.sum(_var, axis=1) * inv_D + rstd = 1 / tl.sqrt(var + eps) + tl.store(Rstd + rows, rstd, mask=row_mask) + rstd = tl.expand_dims(rstd, 1) + + y = x_mean * rstd + w = tl.load(W + cols, mask=col_mask).to(tl.float32) + b = tl.load(B + cols, mask=col_mask).to(tl.float32) + y = y * w[None, :] + b[None, :] + + # Pre-compute sigmoid once to avoid redundant computation + sigmoid_u_block = tl.sigmoid(u_block) + silu_u_block = u_block * sigmoid_u_block + + if MUL_U_ACTIVATION_TYPE == "silu": + y = y * silu_u_block + elif MUL_U_ACTIVATION_TYPE == "sigmoid": + y = y * sigmoid_u_block + else: + y = y * u_block + + if CONCAT_U and SILU_U: + # pyre-fixme[16] + u_block = silu_u_block + + if TRAINING: + # Reuse rows (as int64 for pointer arithmetic) and pre-computed mask_2d + row_offsets_i64 = rows.to(tl.int64) + # Pre-compute loop-invariant values + dropout_scale = 1.0 / (1.0 - dropout_ratio) + offsets = row_offsets_i64[:, None] * stride_mask + cols[None, :] + + if CONCAT_U or CONCAT_X: + # All 2+ mask cases use compressed int8 format - load once + compressed = tl.load(RANDOM_MASK + offsets, mask=mask_2d, other=0).to( + tl.int32 + ) + # Bit 0 is always y_mask + y_keep = (compressed & 1) != 0 + + if CONCAT_U and CONCAT_X: + # 3-mask: (u_mask << 2) | (x_mask << 1) | y_mask + x_keep = (compressed & 2) != 0 + u_keep = (compressed & 4) != 0 + u_block = tl.where(u_keep, u_block * dropout_scale, 0.0) + x_block = tl.where(x_keep, x_block * dropout_scale, 0.0) + elif CONCAT_U: + # 2-mask: (u_mask << 1) | y_mask + u_keep = (compressed & 2) != 0 + u_block = tl.where(u_keep, u_block * dropout_scale, 0.0) + else: # CONCAT_X + # 2-mask: (x_mask << 1) | y_mask + x_keep = (compressed & 2) != 0 + x_block = tl.where(x_keep, x_block * dropout_scale, 0.0) + + y = tl.where(y_keep, y * dropout_scale, 0.0) + else: + # 1-mask: y_mask at bit 0 + y_keep = tl.load(RANDOM_MASK + offsets, mask=mask_2d, other=True) + y = tl.where(y_keep, y * dropout_scale, 0.0) + + if CONCAT_U and CONCAT_X: + Y_block_ptr_u = tl.make_block_ptr( + base=Y, + shape=(N, 3 * D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr_x = tl.make_block_ptr( + base=Y, + shape=(N, 3 * D), + strides=(stride_y, 1), + offsets=(start_row, D), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr_y = tl.make_block_ptr( + base=Y, + shape=(N, 3 * D), + strides=(stride_y, 1), + offsets=(start_row, 2 * D), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + tl.store(Y_block_ptr_u, u_block.to(Y.dtype.element_ty), boundary_check=(0, 1)) + tl.store(Y_block_ptr_x, x_block.to(Y.dtype.element_ty), boundary_check=(0, 1)) + tl.store(Y_block_ptr_y, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + elif CONCAT_U: + Y_block_ptr_u = tl.make_block_ptr( + base=Y, + shape=(N, 2 * D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + Y_block_ptr_y = tl.make_block_ptr( + base=Y, + shape=(N, 2 * D), + strides=(stride_y, 1), + offsets=(start_row, D), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + tl.store(Y_block_ptr_u, u_block.to(Y.dtype.element_ty), boundary_check=(0, 1)) + tl.store(Y_block_ptr_y, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + elif CONCAT_X: + Y_block_ptr_x = tl.make_block_ptr( + base=Y, + shape=(N, 2 * D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + Y_block_ptr_y = tl.make_block_ptr( + base=Y, + shape=(N, 2 * D), + strides=(stride_y, 1), + offsets=(start_row, D), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + tl.store(Y_block_ptr_x, x_block.to(Y.dtype.element_ty), boundary_check=(0, 1)) + tl.store(Y_block_ptr_y, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + else: + Y_block_ptr = tl.make_block_ptr( + base=Y, + shape=(N, D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + tl.store(Y_block_ptr, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.jit +def _ln_mul_dropout_fwd( + X, + U, + Y, + W, + B, + Mean, + Rstd, + D, + eps, + seed, + dropout_ratio, + stride_x, + stride_u, + stride_y, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_U: tl.constexpr, + CONCAT_X: tl.constexpr, + MUL_U_ACTIVATION_TYPE: tl.constexpr, + FAST_DROPOUT: tl.constexpr, +): + row = tl.program_id(0) + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + Y += row.to(tl.int64) * stride_y + cols = tl.arange(0, BLOCK_D) + + # Compute mean + mean = 0.0 + x = tl.load(X + cols, mask=cols < D, other=0.0).to(tl.float32) + mean = tl.sum(x, axis=0) / D + + # Compute variance + _var = tl.zeros([BLOCK_D], dtype=tl.float32) + x_mean = tl.where(cols < D, x - mean, 0.0) + _var += x_mean * x_mean + var = tl.sum(_var, axis=0) / D + rstd = 1 / tl.sqrt(var + eps) + tl.store(Mean + row, mean) + tl.store(Rstd + row, rstd) + + # Normalize and apply linear transformation + mask = cols < D + y = x_mean * rstd + w = tl.load(W + cols, mask=mask).to(tl.float32) + b = tl.load(B + cols, mask=mask).to(tl.float32) + y = y * w + b + u = tl.load(U + cols, mask=cols < D, other=0.0).to(tl.float32) + sigmoid_u = tl.sigmoid(u) + silu_u = u * sigmoid_u + + if MUL_U_ACTIVATION_TYPE == "silu": + y = y * silu_u + elif MUL_U_ACTIVATION_TYPE == "sigmoid": + y = y * sigmoid_u + else: + y = y * u + + if CONCAT_U and SILU_U: + u = silu_u + + if TRAINING: + random_offsets = 3 * row * BLOCK_D + cols + if CONCAT_U and CONCAT_X: + # apply dropout on u + if FAST_DROPOUT: + random_u, random_x, random_y = rand3x(seed, random_offsets) + else: + random_u = tl.rand(seed, random_offsets) + u_keep = random_u > dropout_ratio + u = tl.where(u_keep, u / (1.0 - dropout_ratio), 0.0) + # apply dropout on x + if not FAST_DROPOUT: + random_x = tl.rand(seed, random_offsets + D) + x_keep = random_x > dropout_ratio # pyre-ignore [61] + x = tl.where(x_keep, x / (1.0 - dropout_ratio), 0.0) + # apply dropout on y + if not FAST_DROPOUT: + random_y = tl.rand(seed, random_offsets + 2 * D) + y_keep = random_y > dropout_ratio # pyre-ignore [61] + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + elif CONCAT_U: + # apply dropout on u + if FAST_DROPOUT: + random_u, random_y, _ = rand3x(seed, random_offsets) + else: + random_u = tl.rand(seed, random_offsets) + u_keep = random_u > dropout_ratio + u = tl.where(u_keep, u / (1.0 - dropout_ratio), 0.0) + # apply dropout on y + if not FAST_DROPOUT: + random_y = tl.rand(seed, random_offsets + D) + y_keep = random_y > dropout_ratio # pyre-ignore [61] + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + elif CONCAT_X: + # apply dropout on x + if FAST_DROPOUT: + random_x, random_y, _ = rand3x(seed, random_offsets) + else: + random_x = tl.rand(seed, random_offsets) + x_keep = random_x > dropout_ratio + x = tl.where(x_keep, x / (1.0 - dropout_ratio), 0.0) + # apply dropout on y + if not FAST_DROPOUT: + random_y = tl.rand(seed, random_offsets + D) + y_keep = random_y > dropout_ratio # pyre-ignore [61] + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + else: + random = tl.rand(seed, random_offsets) + y_keep = random > dropout_ratio + # write-back + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + + # Write output + if CONCAT_U and CONCAT_X: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_U: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_X: + tl.store(Y + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + + +@triton.jit +def _ln_mul_dropout_bwd_dx_du_rng( + DX, + DU, + DY, + DW, + DB, + X, + U, + Y, + W, + B, + Mean, + Rstd, + RANDOM_MASK, + stride_dx, + stride_du, + stride_dy, + stride_x, + stride_u, + stride_y, + stride_mask, + D, + eps, + dropout_ratio, + N, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_U: tl.constexpr, + CONCAT_X: tl.constexpr, + MUL_U_ACTIVATION_TYPE: tl.constexpr, + COMPUTE_Y: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + rows_per_tile = N // tile_num + if pid < N % tile_num: + rows_per_tile += 1 + + if rows_per_tile == 0: + return + + cols = tl.arange(0, BLOCK_D) + mask = cols < D + + row = pid + # Pre-compute row and pid as int64 once for initial pointer setup + row_i64 = row.to(tl.int64) + pid_i64 = pid.to(tl.int64) + X += row_i64 * stride_x + U += row_i64 * stride_u + if COMPUTE_Y: + Y += row_i64 * stride_y + DY += row_i64 * stride_dy + DX += row_i64 * stride_dx + DU += row_i64 * stride_du + DW = DW + pid_i64 * D + cols + DB = DB + pid_i64 * D + cols + + # Pre-compute mask pointer offset (all cases use stride_mask for (N, D) shape) + RANDOM_MASK += row_i64 * stride_mask + + partial_dw = tl.zeros((BLOCK_D,), dtype=tl.float32) + partial_db = tl.zeros((BLOCK_D,), dtype=tl.float32) + w = tl.load(W + cols, mask=mask).to(tl.float32) + b = tl.load(B + cols, mask=mask).to(tl.float32) + + dropout_scale = 0.0 + if TRAINING: + dropout_scale = 1.0 / (1.0 - dropout_ratio) + + # Pre-compute inv_D to replace divisions with multiplications (optimization) + inv_D = 1.0 / D + + # Pre-compute tile_num as int64 to avoid repeated conversion in the loop + tile_num_i64 = tile_num.to(tl.int64) + for _ in range(0, rows_per_tile): + # Load data to SRAM + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + if CONCAT_U and CONCAT_X: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + 2 * D + cols, mask=mask, other=0).to(tl.float32) + elif CONCAT_U: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + elif CONCAT_X: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + else: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + if TRAINING: + if CONCAT_U or CONCAT_X: + # All 2+ mask cases use compressed int8 format - load once + compressed = tl.load(RANDOM_MASK + cols, mask=mask, other=0).to( + tl.int32 + ) + dy_keep = (compressed & 1) != 0 # Bit 0 always y_mask + + if CONCAT_U and CONCAT_X: + # Format: (u_mask << 2) | (x_mask << 1) | y_mask + dx_keep = (compressed & 2) != 0 + du_keep = (compressed & 4) != 0 + du = tl.where(du_keep, du * dropout_scale, 0.0) + dx = tl.where(dx_keep, dx * dropout_scale, 0.0) + elif CONCAT_U: + # Format: (u_mask << 1) | y_mask + du_keep = (compressed & 2) != 0 + du = tl.where(du_keep, du * dropout_scale, 0.0) + else: # CONCAT_X + # Format: (x_mask << 1) | y_mask + dx_keep = (compressed & 2) != 0 + dx = tl.where(dx_keep, dx * dropout_scale, 0.0) + dy = tl.where(dy_keep, dy * dropout_scale, 0.0) + else: + # 1-mask: y_mask at bit 0 + dy_keep = tl.load(RANDOM_MASK + cols, mask=mask, other=True) + dy = tl.where(dy_keep, dy * dropout_scale, 0.0) + + mean = tl.load(Mean + row) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = (x - mean) * rstd + u = tl.load(U + cols, mask=mask, other=0).to(tl.float32) + ln = xhat * w + b + du_y = dy * ln + mul_u = u + sig_u = tl.sigmoid(u) + + # Pre-compute commonly used expressions to avoid redundant computation + silu_u = u * sig_u # silu(u) - used multiple times + dsig_u = sig_u * (1.0 - sig_u) # sigmoid derivative - used multiple times + dsilu_u = sig_u + silu_u * ( + 1.0 - sig_u + ) # silu derivative - used multiple times + + if MUL_U_ACTIVATION_TYPE == "silu": + mul_u = silu_u + du_y = dy * ln * dsilu_u + dy = dy * silu_u + elif MUL_U_ACTIVATION_TYPE == "sigmoid": + mul_u = sig_u + du_y = dy * ln * dsig_u + dy = dy * sig_u + else: + dy = dy * u + + du_u = du + if CONCAT_U and SILU_U: + du_u *= dsilu_u + u = silu_u + + du = du_y + du_u + + tl.store(DU + cols, du.to(DU.dtype.element_ty), mask=mask) + + wdy = w * dy + if COMPUTE_Y: + y = ln * mul_u + if TRAINING: + if CONCAT_U: + u = tl.where( + du_keep, # pyre-ignore [61] + u * dropout_scale, + 0.0, + ) + if CONCAT_X: + x = tl.where( + dx_keep, # pyre-ignore [61] + x * dropout_scale, + 0.0, + ) + y = tl.where( + dy_keep, # pyre-ignore [61] + y * dropout_scale, + 0.0, + ) + if CONCAT_U and CONCAT_X: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_U: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_X: + tl.store(Y + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + Y += tile_num_i64 * stride_y + + # Note: xhat and wdy are already 0 outside valid range due to masked loads, + # so no additional tl.where masking is needed before reduction + c1 = tl.sum(xhat * wdy, axis=0) * inv_D + c2 = tl.sum(wdy, axis=0) * inv_D + dx += (wdy - (xhat * c1 + c2)) * rstd + # Write dx + tl.store(DX + cols, dx, mask=mask) + + # Accumulate partial sums for dw/db + partial_dw += dy * xhat + partial_db += dy + X += tile_num_i64 * stride_x + U += tile_num_i64 * stride_u + DY += tile_num_i64 * stride_dy + DX += tile_num_i64 * stride_dx + DU += tile_num_i64 * stride_du + # Increment mask pointer + RANDOM_MASK += tile_num_i64 * stride_mask + row += tile_num + tl.store(DW, partial_dw, mask=mask) + tl.store(DB, partial_db, mask=mask) + + +@triton.jit +def _ln_mul_dropout_bwd_dx_du( + DX, + DU, + DY, + DW, + DB, + X, + U, + Y, + W, + B, + Mean, + Rstd, + stride_dx, + stride_du, + stride_dy, + stride_x, + stride_u, + stride_y, + D, + eps, + seed, + dropout_ratio, + N, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_U: tl.constexpr, + CONCAT_X: tl.constexpr, + MUL_U_ACTIVATION_TYPE: tl.constexpr, + COMPUTE_Y: tl.constexpr, + FAST_DROPOUT: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + rows_per_tile = N // tile_num + if pid < N % tile_num: + rows_per_tile += 1 + + if rows_per_tile == 0: + return + + cols = tl.arange(0, BLOCK_D) + mask = cols < D + + row = pid + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + if COMPUTE_Y: + Y += row.to(tl.int64) * stride_y + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + DU += row.to(tl.int64) * stride_du + DW = DW + pid * D + cols + DB = DB + pid * D + cols + + partial_dw = tl.zeros((BLOCK_D,), dtype=tl.float32) + partial_db = tl.zeros((BLOCK_D,), dtype=tl.float32) + w = tl.load(W + cols, mask=mask).to(tl.float32) + b = tl.load(B + cols, mask=mask).to(tl.float32) + for _idx in range(0, rows_per_tile): + # Load data to SRAM + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + if CONCAT_U and CONCAT_X: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + 2 * D + cols, mask=mask, other=0).to(tl.float32) + elif CONCAT_U: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + elif CONCAT_X: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + else: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + if TRAINING: + random_offsets = 3 * row * BLOCK_D + cols + if CONCAT_U and CONCAT_X: + # apply dropout on du + if FAST_DROPOUT: + random_du, random_dx, random_dy = rand3x(seed, random_offsets) + else: + random_du = tl.rand(seed, random_offsets) + du_keep = random_du > dropout_ratio + du = tl.where(du_keep, du / (1.0 - dropout_ratio), 0.0) + # apply dropout on dx + if not FAST_DROPOUT: + random_dx = tl.rand(seed, random_offsets + D) + dx_keep = random_dx > dropout_ratio # pyre-ignore [61] + dx = tl.where(dx_keep, dx / (1.0 - dropout_ratio), 0.0) + # apply dropout on dy + if not FAST_DROPOUT: + random_dy = tl.rand(seed, random_offsets + 2 * D) + dy_keep = random_dy > dropout_ratio # pyre-ignore [61] + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + elif CONCAT_U: + # apply dropout on du + if FAST_DROPOUT: + random_du, _, random_dy = rand3x(seed, random_offsets) + else: + random_du = tl.rand(seed, random_offsets) + du_keep = random_du > dropout_ratio + du = tl.where(du_keep, du / (1.0 - dropout_ratio), 0.0) + # apply dropout on dy + if not FAST_DROPOUT: + random_dy = tl.rand(seed, random_offsets + D) + dy_keep = random_dy > dropout_ratio # pyre-ignore [61] + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + elif CONCAT_X: + # apply dropout on dx + if FAST_DROPOUT: + _, random_dx, random_dy = rand3x(seed, random_offsets) + else: + random_dx = tl.rand(seed, random_offsets) + dx_keep = random_dx > dropout_ratio # pyre-ignore [61] + dx = tl.where(dx_keep, dx / (1.0 - dropout_ratio), 0.0) + # apply dropout on dy + if not FAST_DROPOUT: + random_dy = tl.rand(seed, random_offsets + D) + dy_keep = random_dy > dropout_ratio # pyre-ignore [61] + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + else: + random = tl.rand(seed, random_offsets) + dy_keep = random > dropout_ratio + # write-back + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + + mean = tl.load(Mean + row) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = (x - mean) * rstd + u = tl.load(U + cols, mask=mask, other=0).to(tl.float32) + ln = xhat * w + b + du_y = dy * ln + mul_u = u + sig_u = tl.sigmoid(u) + + if MUL_U_ACTIVATION_TYPE == "silu": + mul_u = u * sig_u + du_y = dy * ln * (sig_u + u * sig_u * (1.0 - sig_u)) + dy = dy * u * sig_u + elif MUL_U_ACTIVATION_TYPE == "sigmoid": + mul_u = sig_u + du_y = dy * ln * (sig_u * (1.0 - sig_u)) + dy = dy * sig_u + else: + dy = dy * u + + du_u = du + if CONCAT_U: + if SILU_U: + du_u *= sig_u + u * sig_u * (1.0 - sig_u) + u = u * sig_u + + du = du_y + du_u + + tl.store(DU + cols, du.to(DU.dtype.element_ty), mask=mask) + wdy = w * dy + if COMPUTE_Y: + y = ln * mul_u + if TRAINING: + if CONCAT_U: + u = tl.where( + du_keep, # pyre-ignore [61] + u / (1.0 - dropout_ratio), + 0.0, + ) + if CONCAT_X: + x = tl.where( + dx_keep, # pyre-ignore [61] + x / (1.0 - dropout_ratio), + 0.0, + ) + y = tl.where( + dy_keep, # pyre-ignore [61] + y / (1.0 - dropout_ratio), + 0.0, + ) + if CONCAT_U and CONCAT_X: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_U: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + elif CONCAT_X: + tl.store(Y + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + Y += tile_num.to(tl.int64) * stride_y + + xhat = tl.where(mask, xhat, 0.0) + wdy = tl.where(mask, wdy, 0.0) + c1 = tl.sum(xhat * wdy, axis=0) / D + c2 = tl.sum(wdy, axis=0) / D + dx += (wdy - (xhat * c1 + c2)) * rstd + # Write dx + tl.store(DX + cols, dx, mask=mask) + + # Accumulate partial sums for dw/db + partial_dw += dy * xhat + partial_db += dy + X += tile_num.to(tl.int64) * stride_x + U += tile_num.to(tl.int64) * stride_u + DY += tile_num.to(tl.int64) * stride_dy + DX += tile_num.to(tl.int64) * stride_dx + DU += tile_num.to(tl.int64) * stride_du + row += tile_num + tl.store(DW, partial_dw, mask=mask) + tl.store(DB, partial_db, mask=mask) + + +def _get_bwd_dwdb_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [32, 64, 128, 256]: + for num_warps in [8, 16] + ([] if torch.ops.hip else [32]): + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=["D"], +) +@triton.jit +def _ln_mul_dropout_bwd_dwdb( + DW, + DB, + FINAL_DW, + FINAL_DB, + N, + D, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid = tl.program_id(0).to(tl.int64) + cols = pid * BLOCK_D + tl.arange(0, BLOCK_D) + dw = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + db = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask = (rows[:, None] < N) & (cols[None, :] < D) + offs = rows[:, None] * D + cols[None, :] + dw += tl.load(DW + offs, mask=mask, other=0.0) + db += tl.load(DB + offs, mask=mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + sum_db = tl.sum(db, axis=0) + tl.store(FINAL_DW + cols, sum_dw.to(FINAL_DW.dtype.element_ty), mask=cols < D) + tl.store(FINAL_DB + cols, sum_db.to(FINAL_DB.dtype.element_ty), mask=cols < D) + + +def _create_dropout_mask( + N: int, + D: int, + BLOCK_D: int, + concat_u: bool, + concat_x: bool, + dropout_ratio: float, + seed: int, + device: torch.device, +) -> torch.Tensor: + """Create dropout mask tensor for layer norm mul dropout. + + Args: + N: Number of rows + D: Feature dimension + BLOCK_D: Block size for D dimension + concat_u: Whether to concatenate u + concat_x: Whether to concatenate x + dropout_ratio: Dropout ratio + seed: Random seed + device: Device to create tensor on + + Returns: + random_mask: (N, D) int8 tensor. Mask j at bit j. + + Bit layout: y = val & 1, x = val & 2, u = val & 4. + """ + num_masks = 1 + int(concat_u) + int(concat_x) + # Torch uses 1 byte for bool internally, same as int8, so always use int8. + random_mask = torch.empty([N, D], dtype=torch.int8, device=device) + _generate_random_mask[(triton.cdiv(N, 4),)]( + random_mask, + N, + dropout_ratio, + seed, + D, # pyre-ignore[6] + random_mask.stride(0), # pyre-ignore[6] + BLOCK_D, # pyre-fixme[6]: Triton constexpr param + num_masks, # pyre-ignore[6]: NUM_MASKS constexpr + ) + return random_mask + + +@maybe_register_custom_op( + "generative_recommenders::_triton_layer_norm_mul_dropout_fwd_impl", mutates_args=() +) +def _triton_layer_norm_mul_dropout_fwd_impl( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + seed: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Internal implementation that returns only tensors for custom_op compatibility. + + Returns (y, mean, rstd, random_mask) where random_mask is empty when not used. + """ + N, D = x.shape + + if concat_u and concat_x: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + elif concat_x: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + mean = torch.empty((N,), dtype=torch.float32, device=x.device) + rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + if N == 0: + return y, mean, rstd, torch.empty(0, dtype=x.dtype, device=x.device) + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D: int = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + num_warps: int = min(max(BLOCK_D // 256, 1), 8) + random_mask: torch.Tensor = torch.empty(0, dtype=x.dtype, device=x.device) + # Separating RNG from the ln_mul_dropout kernel lets us batch multiple rows per + # program (autotuned _ln_mul_dropout_fwd_rng) and reuse the precomputed mask in the + # backward, instead of launching one program per row with fused RNG. This is a large + # win on Blackwell (sm_100) and AMD MI350 (gfx950); other GPUs keep the fused path. + # Extended to support concat_u + concat_x for mask reuse optimization + if ( + not FUSE_OUTPUT_LN_RNG_BLACKWELL + and use_separated_rng_ln_mul_dropout() + and training + ): + random_mask = _create_dropout_mask( + N=N, + D=D, + BLOCK_D=BLOCK_D, + concat_u=concat_u, + concat_x=concat_x, + dropout_ratio=dropout_ratio, + seed=seed, + device=x.device, + ) + + def grid(META): + return (triton.cdiv(N, META["BLOCK_N"]),) + + # pyre-ignore[28] + _ln_mul_dropout_fwd_rng[grid]( + x, + u, + y, + weight, + bias, + mean, + rstd, + random_mask, + N, + D, + eps, + dropout_ratio, + x.stride(0), + u.stride(0), + y.stride(0), + random_mask.stride(0), + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_U=concat_u, + CONCAT_X=concat_x, + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + ) + + else: + # Default path: fused RNG generation + # Mask cannot be saved with fused RNG - it's generated inline in the kernel + # pyre-ignore[28] + _ln_mul_dropout_fwd[(N,)]( + x, + u, + y, + weight, + bias, + mean, + rstd, + D, + eps, + seed, + dropout_ratio, + x.stride(0), + u.stride(0), + y.stride(0), + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_U=concat_u, + CONCAT_X=concat_x, + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + FAST_DROPOUT=COMPUTE_OUTPUT_LN_FAST_DROPOUT, + num_warps=num_warps, + ) + return y, mean, rstd, random_mask + + +@_triton_layer_norm_mul_dropout_fwd_impl.register_fake +def _triton_layer_norm_mul_dropout_fwd_impl_fake( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + seed: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fake implementation for FakeTensor tracing.""" + N, D = x.shape + if concat_u and concat_x: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + elif concat_x: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + mean = torch.empty((N,), dtype=torch.float32, device=x.device) + rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + random_mask = torch.empty(0, dtype=x.dtype, device=x.device) + return y, mean, rstd, random_mask + + +def triton_layer_norm_mul_dropout_fwd( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + seed: Optional[int] = None, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, Optional[torch.Tensor] +]: # y, mean, rstd, BLOCK_D, num_warps, seed, random_mask + """Forward pass for layer norm + mul + dropout. + + Args: + x: Input tensor of shape (N, D) + u: Second input tensor of shape (N, D) + weight: Layer norm weight of shape (D,) + bias: Layer norm bias of shape (D,) + eps: Layer norm epsilon + dropout_ratio: Dropout probability + training: Whether in training mode + silu_u: Whether to apply SiLU to u before concatenation + concat_u: Whether to concatenate u to output + concat_x: Whether to concatenate x to output + mul_u_activation_type: Activation type for u multiplication + seed: Random seed for dropout + + Returns: + Tuple of (y, mean, rstd, BLOCK_D, num_warps, seed, random_mask) + - random_mask is None when using fused RNG path (non-SM100+) + - random_mask is always returned when using separate RNG path (SM100+) + for reuse in backward pass (avoids redundant mask generation) + """ + assert x.dim() == 2 + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == D + assert bias.numel() == D + + if N == 0: + D = x.shape[1] + if concat_u and concat_x: + y = torch.empty((0, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u or concat_x: + y = torch.empty((0, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + return ( + y, + torch.empty((N,), dtype=torch.float32, device=x.device), + torch.empty((N,), dtype=torch.float32, device=x.device), + 0, + 0, + 0, + None, + ) + + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D: int = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + if seed is None and training: + # pyre-ignore[9]: torch.randint with dtype=int64 always returns int + seed = torch.randint(low=0, high=2**62, size=(1,), dtype=torch.int64).item() + num_warps: int = min(max(BLOCK_D // 256, 1), 8) + + # Call internal implementation + y, mean, rstd, random_mask_tensor = _triton_layer_norm_mul_dropout_fwd_impl( + x, + u, + weight, + bias, + eps, + dropout_ratio, + training, + silu_u, + concat_u, + concat_x, + mul_u_activation_type, + seed if seed is not None else 0, + ) + + # Convert empty tensor back to None + random_mask: Optional[torch.Tensor] = ( + random_mask_tensor if random_mask_tensor.numel() > 0 else None + ) + + return y, mean, rstd, BLOCK_D, num_warps, seed, random_mask # pyre-ignore[7] + + +@maybe_register_custom_op( + "generative_recommenders::_triton_layer_norm_mul_dropout_bwd_impl", mutates_args=() +) +def _triton_layer_norm_mul_dropout_bwd_impl( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: int, + silu_u: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + compute_y: bool, + random_mask: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Internal implementation that returns only tensors for custom_op compatibility. + + When compute_y is False, y is returned as an empty tensor. + random_mask with numel() == 0 means no mask (fused RNG path). + """ + N, D = x.shape + if compute_y: + if concat_u and concat_x: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + elif concat_x: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + else: + y = torch.empty(0, dtype=x.dtype, device=x.device) + + if N == 0: + return ( + torch.zeros_like(x), + torch.zeros_like(u), + torch.zeros((D,), dtype=weight.dtype, device=x.device), + torch.zeros((D,), dtype=weight.dtype, device=x.device), + y, + ) + dx = torch.empty_like(x) + du = torch.empty_like(u) + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + tile_num = max(1, min(sms * 64, N // 4)) + _dweight = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + _dbias = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + + # Use separated RNG when random_mask is provided (from forward pass on SM100+ path) + has_random_mask = random_mask.numel() > 0 + if has_random_mask: + # pyre-ignore[28] + _ln_mul_dropout_bwd_dx_du_rng[(tile_num,)]( + dx, + du, + dy, + _dweight, + _dbias, + x, + u, + y if compute_y else None, + weight, + bias, + mean, + rstd, + random_mask, + dx.stride(0), + du.stride(0), + dy.stride(0), + x.stride(0), + u.stride(0), + y.stride(0) if compute_y else 0, + random_mask.stride(0), + D, + eps, + dropout_ratio, + N=N, + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_U=concat_u, + CONCAT_X=concat_x, + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + COMPUTE_Y=compute_y, + num_warps=num_warps, + ) + + else: + # pyre-ignore[28] + _ln_mul_dropout_bwd_dx_du[(tile_num,)]( + dx, + du, + dy, + _dweight, + _dbias, + x, + u, + y if compute_y else None, + weight, + bias, + mean, + rstd, + dx.stride(0), + du.stride(0), + dy.stride(0), + x.stride(0), + u.stride(0), + y.stride(0) if compute_y else 0, + D, + eps, + seed, + dropout_ratio, + N=N, + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_U=concat_u, + CONCAT_X=concat_x, + MUL_U_ACTIVATION_TYPE=mul_u_activation_type, + COMPUTE_Y=compute_y, + FAST_DROPOUT=COMPUTE_OUTPUT_LN_FAST_DROPOUT, + num_warps=num_warps, + ) + + def grid(META): + return (triton.cdiv(D, META["BLOCK_D"]),) + + blocks = triton.next_power_of_2(sms * 4) + BLOCK_D_bwd = triton.next_power_of_2(triton.cdiv(D, blocks)) + BLOCK_D_bwd = min(max(BLOCK_D_bwd, 4), 128) + _ln_mul_dropout_bwd_dwdb[grid]( + _dweight, + _dbias, + dweight, + dbias, + tile_num, + D, + BLOCK_D=BLOCK_D_bwd, + ) + return dx, du, dweight, dbias, y + + +@_triton_layer_norm_mul_dropout_bwd_impl.register_fake +def _triton_layer_norm_mul_dropout_bwd_impl_fake( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: int, + silu_u: bool, + concat_u: bool, + concat_x: bool, + mul_u_activation_type: str, + compute_y: bool, + random_mask: torch.Tensor, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]: + """Fake implementation for FakeTensor tracing.""" + N, D = x.shape + dx = torch.empty_like(x) + du = torch.empty_like(u) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + if compute_y: + if concat_u and concat_x: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + elif concat_u: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + elif concat_x: + y = torch.empty((N, 2 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + else: + y = torch.empty(0, dtype=x.dtype, device=x.device) + return dx, du, dweight, dbias, y + + +def triton_layer_norm_mul_dropout_bwd( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: Optional[int] = None, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + compute_y: bool = False, + random_mask: Optional[torch.Tensor] = None, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor] +]: + N, D = x.shape + + # Use empty tensor as sentinel for no random_mask + random_mask_tensor = ( + random_mask + if random_mask is not None + else torch.empty(0, dtype=x.dtype, device=x.device) + ) + + dx, du, dweight, dbias, y_tensor = _triton_layer_norm_mul_dropout_bwd_impl( + dy, + x, + u, + weight, + bias, + mean, + rstd, + BLOCK_D, + num_warps, + eps, + training, + dropout_ratio, + seed if seed is not None else 0, + silu_u, + concat_u, + concat_x, + mul_u_activation_type, + compute_y, + random_mask_tensor, + ) + + # Convert empty tensor back to None + y: Optional[torch.Tensor] = y_tensor if compute_y else None + return dx, du, dweight, dbias, y + + +class LayerNormMulDropoutFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + seed: Optional[int] = None, + ) -> torch.Tensor: + if dropout_ratio == 0.0: + # skip dropout computation if dropout ratio is 0 + training = False + # skipping supporting concat_u and concat_x separately here because seems like this code path is only used in v1 of hstu_linear + concat_u, concat_x = concat_ux, concat_ux + + # Call forward function which generates and returns random_mask + # On SM100+ path, random_mask is always returned for backward reuse + # On fused RNG path, random_mask is None (mask generated inline in kernel) + y, mean, rstd, BLOCK_D, num_warps, returned_seed, random_mask = ( + triton_layer_norm_mul_dropout_fwd( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_u=concat_u, + concat_x=concat_x, + seed=seed, + ) + ) + + # Save tensors for backward pass + # When random_mask is generated (SM100+ path), always save it for reuse + # in backward pass. This avoids redundant _generate_random_mask execution. + if random_mask is not None: + ctx.save_for_backward(x, u, weight, bias, mean, rstd, random_mask) + ctx.has_random_mask = True + else: + ctx.save_for_backward(x, u, weight, bias, mean, rstd) + ctx.has_random_mask = False + + ctx.BLOCK_D = BLOCK_D + ctx.num_warps = num_warps + ctx.eps = eps + ctx.seed = returned_seed + ctx.training = training + ctx.concat_ux = concat_ux + ctx.silu_u = silu_u + ctx.dropout_ratio = dropout_ratio + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + ]: + # Extract saved tensors including optional random mask + if ctx.has_random_mask: + x, u, weight, bias, mean, rstd, random_mask = ctx.saved_tensors + else: + x, u, weight, bias, mean, rstd = ctx.saved_tensors + random_mask = None + + dx, du, dweight, dbias, _ = triton_layer_norm_mul_dropout_bwd( + dy=dy, + x=x, + u=u, + weight=weight, + bias=bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_u=ctx.concat_ux, + concat_x=ctx.concat_ux, + compute_y=False, + random_mask=random_mask, # Pass saved mask to backward + ) + return dx, du, dweight, dbias, None, None, None, None, None, None + + +@triton.jit +def _group_norm_mul_dropout_fwd( + X, + U, + Y, + W, + B, + Mean, + Rstd, + D, + Heads, + eps, + seed, + dropout_ratio, + stride_x, + stride_u, + stride_y, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_H: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_UX: tl.constexpr, +): + row = tl.program_id(0) + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + Y += row.to(tl.int64) * stride_y + cols = tl.arange(0, BLOCK_D) + heads = tl.arange(0, BLOCK_H) + offsets = heads[:, None] * D + cols[None, :] + mask_h = heads < Heads + mask_c = cols < D + mask = mask_c[None, :] & mask_h[:, None] + + # Compute mean + mean = 0.0 + x = tl.load(X + offsets, mask=mask, other=0.0).to(tl.float32) + mean = tl.sum(x, axis=1) / D + mean = tl.ravel(mean) + + # Compute variance + _var = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + x_mean = tl.where(mask, x - mean[:, None], 0.0) + _var += x_mean * x_mean + var = tl.sum(_var, axis=1) / D + var = tl.ravel(var) + rstd = 1 / tl.sqrt(var + eps) + tl.store(Mean + row * Heads + heads, mean, mask=mask_h) + tl.store(Rstd + row * Heads + heads, rstd, mask=mask_h) + + # Normalize and apply linear transformation + y = x_mean * rstd[:, None] # pyre-ignore [16] + w = tl.load(W + heads, mask=mask_h).to(tl.float32) + b = tl.load(B + heads, mask=mask_h).to(tl.float32) + y = y * w[:, None] + b[:, None] + u = tl.load(U + offsets, mask=mask, other=0.0).to(tl.float32) + if SILU_U: + u = u * tl.sigmoid(u) + y = y * u + + if TRAINING: + if CONCAT_UX: + random_offsets = row * 3 * D * Heads + offsets + # apply dropout on u + random_u = tl.rand(seed, random_offsets) + u_keep = random_u > dropout_ratio + u = tl.where(u_keep, u / (1.0 - dropout_ratio), 0.0) + # apply dropout on x + random_x = tl.rand(seed, random_offsets + Heads * D) + x_keep = random_x > dropout_ratio + x = tl.where(x_keep, x / (1.0 - dropout_ratio), 0.0) + # apply dropout on y + random_y = tl.rand(seed, random_offsets + 2 * Heads * D) + y_keep = random_y > dropout_ratio + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + else: + random_offsets = row * D * Heads + offsets + random = tl.rand(seed, random_offsets) + y_keep = random > dropout_ratio + # write-back + y = tl.where(y_keep, y / (1.0 - dropout_ratio), 0.0) + + # Write output + if CONCAT_UX: + tl.store(Y + offsets, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + Heads * D + offsets, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * Heads * D + offsets, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + offsets, y.to(Y.dtype.element_ty), mask=mask) + + +@triton.jit +def _group_norm_mul_dropout_bwd_dx_du( + DX, + DU, + DY, + DW, + DB, + X, + U, + Y, + W, + B, + Mean, + Rstd, + stride_dx, + stride_du, + stride_dy, + stride_x, + stride_u, + stride_y, + D, + Heads, + eps, + seed, + dropout_ratio, + SILU_U: tl.constexpr, + GROUP_N: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_H: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_UX: tl.constexpr, + COMPUTE_Y: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.arange(0, BLOCK_D) + off_heads = tl.arange(0, BLOCK_H) + mask_c = cols < D + mask_h = off_heads < Heads + mask = mask_c[None, :] & mask_h[:, None] + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + DU += row.to(tl.int64) * stride_du + offsets = off_heads[:, None] * D + cols[None, :] + + # Load data to SRAM + x = tl.load(X + offsets, mask=mask, other=0).to(tl.float32) + if CONCAT_UX: + du = tl.load(DY + offsets, mask=mask, other=0).to(tl.float32) + dx = tl.load(DY + Heads * D + offsets, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + 2 * Heads * D + offsets, mask=mask, other=0).to(tl.float32) + else: + du = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + dx = tl.zeros([BLOCK_H, BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + offsets, mask=mask, other=0).to(tl.float32) + if TRAINING: + if CONCAT_UX: + random_offsets = row * 3 * D * Heads + offsets + # apply dropout on du + random_du = tl.rand(seed, random_offsets) + du_keep = random_du > dropout_ratio + du = tl.where(du_keep, du / (1.0 - dropout_ratio), 0.0) + # apply dropout on dx + random_dx = tl.rand(seed, random_offsets + Heads * D) + dx_keep = random_dx > dropout_ratio + dx = tl.where(dx_keep, dx / (1.0 - dropout_ratio), 0.0) + # apply dropout on dy + random_dy = tl.rand(seed, random_offsets + 2 * Heads * D) + dy_keep = random_dy > dropout_ratio + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + else: + random_offsets = row * D * Heads + offsets + random = tl.rand(seed, random_offsets) + dy_keep = random > dropout_ratio + # write-back + dy = tl.where(dy_keep, dy / (1.0 - dropout_ratio), 0.0) + + mean = tl.load(Mean + row * Heads + off_heads) + rstd = tl.load(Rstd + row * Heads + off_heads) + + # Compute dx + xhat = (x - mean[:, None]) * rstd[:, None] + w = tl.load(W + off_heads, mask=mask_h).to(tl.float32) + b = tl.load(B + off_heads, mask=mask_h).to(tl.float32) + u = tl.load(U + offsets, mask=mask, other=0).to(tl.float32) + ln = xhat * w[:, None] + b[:, None] + du += dy * ln + if SILU_U: + sig_u = tl.sigmoid(u) + silu_u = u * sig_u + du = du * sig_u * (1 + u - silu_u) + u = silu_u + tl.store(DU + offsets, du.to(DU.dtype.element_ty), mask=mask) + dy = dy * u + wdy = w[:, None] * dy + if COMPUTE_Y: + Y += row.to(tl.int64) * stride_y + y = ln * u + if TRAINING: + if CONCAT_UX: + u = tl.where( + du_keep, # pyre-ignore [61] + u / (1.0 - dropout_ratio), + 0.0, + ) + x = tl.where( + dx_keep, # pyre-ignore [61] + x / (1.0 - dropout_ratio), + 0.0, + ) + y = tl.where( + dy_keep, # pyre-ignore [61] + y / (1.0 - dropout_ratio), + 0.0, + ) + else: + y = tl.where( + dy_keep, # pyre-ignore [61] + y / (1.0 - dropout_ratio), + 0.0, + ) + if CONCAT_UX: + tl.store(Y + offsets, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + Heads * D + offsets, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * Heads * D + offsets, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + offsets, y.to(Y.dtype.element_ty), mask=mask) + + xhat = tl.where(mask, xhat, 0.0) + wdy = tl.where(mask, wdy, 0.0) + c1 = tl.sum(xhat * wdy, axis=1) / D + c2 = tl.sum(wdy, axis=1) / D + dx += (wdy - (xhat * c1[:, None] + c2[:, None])) * rstd[:, None] + # Write dx + tl.store(DX + offsets, dx, mask=mask) + + # Offset locks and weights/biases gradient pointer for parallel reduction + lock_id = row % GROUP_N + DW = DW + lock_id * Heads + off_heads + DB = DB + lock_id * Heads + off_heads + # Accumulate partial sums for dw/db + partial_dw = tl.sum(dy * xhat, axis=1) + partial_dw = tl.ravel(partial_dw) + partial_db = tl.sum(dy, axis=1) + partial_db = tl.ravel(partial_db) + tl.atomic_add( + DW, + partial_dw, + mask=mask_h, + sem="relaxed", + ) + tl.atomic_add( + DB, + partial_db, + mask=mask_h, + sem="relaxed", + ) + + +def triton_group_norm_mul_dropout_fwd( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, int, int, int, int +]: # y, mean, rstd, BLOCK_D, BLOCK_H, num_warps, seed + assert x.dim() == 2 + assert x.shape == u.shape + assert x.shape[1] == num_heads * linear_dim + x = switch_to_contiguous_if_needed(x) + u = switch_to_contiguous_if_needed(u) + N, _ = x.shape + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == num_heads + assert bias.numel() == num_heads + + if concat_ux: + y = torch.empty((N, 3 * num_heads * linear_dim), dtype=x.dtype, device=x.device) + else: + y = torch.empty((N, num_heads * linear_dim), dtype=x.dtype, device=x.device) + mean = torch.empty((N * num_heads,), dtype=torch.float32, device=x.device) + rstd = torch.empty((N * num_heads,), dtype=torch.float32, device=x.device) + if N == 0: + return y, mean, rstd, 0, 0, 0, 0 + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D: int = triton.next_power_of_2(linear_dim) + BLOCK_H: int = triton.next_power_of_2(num_heads) + if BLOCK_D * BLOCK_H > MAX_FUSED_SIZE: + raise RuntimeError( + "This group norm doesn't support num_heads * linear_dim >= 64KB." + ) + + if seed is None: + seed = torch.randint(low=0, high=2**62, size=(1,), dtype=torch.int64).item() + num_warps: int = min(max(BLOCK_D * BLOCK_H // 256, 1), 8) + # pyre-ignore[28] + _group_norm_mul_dropout_fwd[(N,)]( + x, + u, + y, + weight, + bias, + mean, + rstd, + linear_dim, + num_heads, + eps, + seed, + dropout_ratio, + x.stride(0), + u.stride(0), + y.stride(0), + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + BLOCK_H=BLOCK_H, + TRAINING=training, + CONCAT_UX=concat_ux, + num_warps=num_warps, + ) + return y, mean, rstd, BLOCK_D, BLOCK_H, num_warps, seed # pyre-ignore [7] + + +def triton_group_norm_mul_dropout_bwd( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + BLOCK_H: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: Optional[int] = None, + silu_u: bool = False, + concat_ux: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + compute_y: bool = False, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor] +]: + y = None + N, dim = x.shape + if compute_y: + if concat_ux: + y = torch.empty( + (N, 3 * num_heads * linear_dim), dtype=x.dtype, device=x.device + ) + else: + y = torch.empty((N, num_heads * linear_dim), dtype=x.dtype, device=x.device) + if N == 0: + return ( + torch.zeros_like(x), + torch.zeros_like(u), + torch.zeros_like(weight), + torch.zeros_like(bias), + y, + ) + dx = torch.empty_like(x) + du = torch.empty_like(u) + if dim <= 1024: + GROUP_N = 256 * 8 + elif dim <= 4096: + GROUP_N = 128 * 8 + elif dim <= 8192: + GROUP_N = 96 * 8 + else: + GROUP_N = 64 * 8 + GROUP_N = N if GROUP_N > N else GROUP_N + _dweight = torch.zeros((GROUP_N, num_heads), dtype=torch.float32, device=x.device) + _dbias = torch.zeros((GROUP_N, num_heads), dtype=torch.float32, device=x.device) + dweight = torch.empty((num_heads,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((num_heads,), dtype=weight.dtype, device=x.device) + # pyre-ignore[28] + _group_norm_mul_dropout_bwd_dx_du[(N,)]( + dx, + du, + dy, + _dweight, + _dbias, + x, + u, + y, + weight, + bias, + mean, + rstd, + dx.stride(0), + du.stride(0), + dy.stride(0), + x.stride(0), + u.stride(0), + y.stride(0) if compute_y else 0, # pyre-ignore [16] + linear_dim, + num_heads, + eps, + seed, + dropout_ratio, + SILU_U=silu_u, + GROUP_N=GROUP_N, + BLOCK_D=BLOCK_D, + BLOCK_H=BLOCK_H, + TRAINING=training, + CONCAT_UX=concat_ux, + COMPUTE_Y=compute_y, + num_warps=num_warps, + ) + _group_norm_bwd_dwdb[(num_heads,)]( + _dweight, + _dbias, + dweight, + dbias, + GROUP_N, + ) + return dx, du, dweight, dbias, y + + +def _get_bwd_dwdb_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [32, 64, 128, 256]: + for num_warps in [8, 16] + ([] if torch.ops.hip else [32]): + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=[], +) +@triton.jit +def _group_norm_bwd_dwdb( + DW, + DB, + FINAL_DW, + FINAL_DB, + N, + BLOCK_N: tl.constexpr, +): + col = tl.program_id(0) + num_heads = tl.num_programs(0) + dw = tl.zeros((BLOCK_N,), dtype=tl.float32) + db = tl.zeros((BLOCK_N,), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + mask = rows < N + offs = rows * num_heads + col + dw += tl.load(DW + offs, mask=mask, other=0.0) + db += tl.load(DB + offs, mask=mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + sum_db = tl.sum(db, axis=0) + tl.store(FINAL_DW + col, sum_dw.to(FINAL_DW.dtype.element_ty)) + tl.store(FINAL_DB + col, sum_db.to(FINAL_DB.dtype.element_ty)) + + +class GroupNormMulDropoutFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, + ) -> torch.Tensor: + y, mean, rstd, BLOCK_D, BLOCK_H, num_warps, seed = ( + triton_group_norm_mul_dropout_fwd( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_ux=concat_ux, + num_heads=num_heads, + linear_dim=linear_dim, + seed=seed, + ) + ) + ctx.save_for_backward(x, u, weight, bias, mean, rstd) + ctx.BLOCK_D = BLOCK_D + ctx.BLOCK_H = BLOCK_H + ctx.num_warps = num_warps + ctx.eps = eps + ctx.seed = seed + ctx.training = training + ctx.silu_u = silu_u + ctx.concat_ux = concat_ux + ctx.dropout_ratio = dropout_ratio + ctx.num_heads = num_heads + ctx.linear_dim = linear_dim + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + None, + None, + ]: + x, u, weight, bias, mean, rstd = ctx.saved_tensors + dx, du, dweight, dbias, _ = triton_group_norm_mul_dropout_bwd( + dy=dy, + x=x, + u=u, + weight=weight, + bias=bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + BLOCK_H=ctx.BLOCK_H, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_ux=ctx.concat_ux, + num_heads=ctx.num_heads, + linear_dim=ctx.linear_dim, + compute_y=False, + ) + return ( + dx, + du, + dweight, + dbias, + None, + None, + None, + None, + None, + None, + None, + None, + ) + + +class HSTUComputeOutputFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + attn: torch.Tensor, + u: torch.Tensor, + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + output_weight: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, + recompute_y_in_backward: bool = False, + ) -> torch.Tensor: + if dropout_ratio == 0.0: + training = False + + if group_norm: + y, mean, rstd, BLOCK_D, BLOCK_H, num_warps, seed = ( + triton_group_norm_mul_dropout_fwd( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_ux=concat_u and concat_x, + num_heads=num_heads, + linear_dim=linear_dim, + seed=seed, + ) + ) + ctx.BLOCK_H = BLOCK_H + random_mask = None + else: + y, mean, rstd, BLOCK_D, num_warps, seed, random_mask = ( + triton_layer_norm_mul_dropout_fwd( + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_u=concat_u, + concat_x=concat_x, + seed=seed, + ) + ) + + out = maybe_triton_addmm_fwd(x=y, w=output_weight, y=x) + + saved_tensors = [attn, u, norm_weight, norm_bias, mean, rstd, output_weight] + if not recompute_y_in_backward: + saved_tensors.append(y) + # Save random_mask for reuse in backward pass (avoids regenerating mask) + # When random_mask is available (SM100+ path), always save it. + if random_mask is not None: + saved_tensors.append(random_mask) + ctx.has_random_mask = True + else: + ctx.has_random_mask = False + ctx.save_for_backward(*saved_tensors) + ctx.BLOCK_D = BLOCK_D + ctx.num_warps = num_warps + ctx.eps = eps + ctx.seed = seed + ctx.training = training + ctx.concat_u = concat_u + ctx.concat_x = concat_x + ctx.dropout_ratio = dropout_ratio + ctx.num_heads = num_heads + ctx.linear_dim = linear_dim + ctx.group_norm = group_norm + ctx.recompute_y_in_backward = recompute_y_in_backward + ctx.silu_u = silu_u + ctx.mul_u_activation_type = mul_u_activation_type + return out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dout: torch.Tensor + ) -> Tuple[ + torch.Tensor, # dattn + torch.Tensor, # du + torch.Tensor, # dx + torch.Tensor, # d_norm_weight + torch.Tensor, # d_norm_bias + torch.Tensor, # d_output_weight + None, # eps + None, # dropout_ratio + None, # training + None, # silu_u + None, # concat_u + None, # concat_x + None, # mul_u_activation_type + None, # group_norm + None, # num_heads + None, # linear_dim + None, # seed + None, # recompute_y_in_backward + ]: + attn, u, norm_weight, norm_bias, mean, rstd, output_weight = ctx.saved_tensors[ + :7 + ] + # Extract optional saved tensors based on flags + next_idx = 7 + if not ctx.recompute_y_in_backward: + saved_y = ctx.saved_tensors[next_idx] + next_idx += 1 + else: + saved_y = None + if ctx.has_random_mask: + random_mask = ctx.saved_tensors[next_idx] + else: + random_mask = None + dy = torch.mm(dout, output_weight.t()) + + if ctx.group_norm: + dattn, du, d_norm_weight, d_norm_bias, y = ( + triton_group_norm_mul_dropout_bwd( + dy=dy, + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + BLOCK_H=ctx.BLOCK_H, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_ux=ctx.concat_u and ctx.concat_x, + num_heads=ctx.num_heads, + linear_dim=ctx.linear_dim, + compute_y=ctx.recompute_y_in_backward, + ) + ) + else: + dattn, du, d_norm_weight, d_norm_bias, y = ( + triton_layer_norm_mul_dropout_bwd( + dy=dy, + x=attn, + u=u, + weight=norm_weight, + bias=norm_bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_u=ctx.concat_u, + concat_x=ctx.concat_x, + mul_u_activation_type=ctx.mul_u_activation_type, + compute_y=ctx.recompute_y_in_backward, + random_mask=random_mask, + ) + ) + if not ctx.recompute_y_in_backward: + y = saved_y + d_output_weight = torch.mm(y.t(), dout) + return ( + dattn, + du, + dout, + d_norm_weight, + d_norm_bias, + d_output_weight, + None, # eps + None, # dropout_ratio + None, # training + None, # silu_u + None, # concat_u + None, # concat_x + None, # mul_u_activation_type + None, # group_norm + None, # num_heads + None, # linear_dim + None, # seed + None, # recompute_y_in_backward + ) + + +@triton.jit +def _helion_ln_mul_dropout_fwd( + x, + weight, + bias, + u, + y, + mean, + rstd, + eps, + seed, + dropout_ratio, + D: tl.constexpr, + stride_x: tl.constexpr, + stride_u: tl.constexpr, + stride_y: tl.constexpr, + BLOCK_D: tl.constexpr, + CONCAT_UX: tl.constexpr, + SILU_U: tl.constexpr, + TRAINING: tl.constexpr, +): + row = tl.program_id(0) + x += row.to(tl.int64) * stride_x + u += row.to(tl.int64) * stride_u + y += row.to(tl.int64) * stride_y + cols = tl.arange(0, BLOCK_D) + mask = cols < D + + # Load input + x_val = tl.load(x + cols, mask=mask, other=0.0).to(tl.float32) + + # Precompute inverse of D for faster computation + inv_D = 1.0 / D + + # Compute mean + mean_val = tl.sum(x_val, axis=0) * inv_D + + # Center the data + x_mean = tl.where(mask, x_val - mean_val, 0.0) + + # Compute variance + var = tl.sum(x_mean * x_mean, axis=0) * inv_D + + # Compute reciprocal standard deviation + # pyre-fixme[16] + rstd_val = libdevice.rsqrt(var + eps) + + # Normalize + y_norm = x_mean * rstd_val + + # Apply weight and bias + w = tl.load(weight + cols, mask=mask, other=0.0).to(tl.float32) + b = tl.load(bias + cols, mask=mask, other=0.0).to(tl.float32) + y_ln = y_norm * w + b + + # Load u and optionally apply SiLU activation + u_val = tl.load(u + cols, mask=mask, other=0.0).to(tl.float32) + if SILU_U: + u_processed = u_val * tl.sigmoid(u_val) + else: + u_processed = u_val + + y_out = y_ln * u_processed + + if TRAINING: + # Compute dropout scale + # pyre-fixme[16] + dropout_scale = fast_dividef(1.0, 1.0 - dropout_ratio) + + if CONCAT_UX: + # Generate dropout masks + random_offsets = 3 * row * BLOCK_D + cols + random_u, random_x, random_y = rand3x(seed, random_offsets) + + u_keep = random_u > dropout_ratio + x_keep = random_x > dropout_ratio + y_keep = random_y > dropout_ratio + + # Apply dropout to u, x, y + u_output = tl.where(u_keep, u_processed * dropout_scale, 0.0) + x_output = tl.where(x_keep, x_val * dropout_scale, 0.0) + y_output = tl.where(y_keep, y_out * dropout_scale, 0.0) + else: + # Generate dropout mask for y + random_offsets = row * BLOCK_D + cols + random_y = tl.rand(seed, random_offsets) + y_keep = random_y > dropout_ratio + + # Apply dropout to y + y_output = tl.where(y_keep, y_out * dropout_scale, 0.0) + else: + if CONCAT_UX: + u_output = u_processed + x_output = x_val + y_output = y_out + + # Store outputs + if CONCAT_UX: + tl.store(y + cols, u_output.to(y.dtype.element_ty), mask=mask) + tl.store(y + D + cols, x_output.to(y.dtype.element_ty), mask=mask) + tl.store(y + 2 * D + cols, y_output.to(y.dtype.element_ty), mask=mask) + else: + tl.store(y + cols, y_output.to(y.dtype.element_ty), mask=mask) + + # Store mean and rstd + tl.store(mean + row, mean_val) + tl.store(rstd + row, rstd_val) + + +def helion_layer_norm_mul_dropout_fwd( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + seed: Optional[int] = None, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, int, int, int +]: # y, mean, rstd, BLOCK_D, num_warps, seed + N, D = x.shape + + if seed is None: + seed = torch.randint(low=0, high=2**62, size=(1,), dtype=torch.int64).item() + + if concat_ux: + y = torch.empty([N, 3 * D], dtype=x.dtype, device=x.device) + else: + y = torch.empty([N, D], dtype=x.dtype, device=x.device) + mean = torch.empty([N], dtype=torch.float32, device=x.device) + rstd = torch.empty([N], dtype=torch.float32, device=x.device) + + BLOCK_D = triton.next_power_of_2(D) + # pyre-ignore[28] + _helion_ln_mul_dropout_fwd[(N,)]( + x, + weight, + bias, + u, + y, + mean, + rstd, + eps, + seed, + dropout_ratio, + D, + x.stride(0), + u.stride(0), + y.stride(0), + BLOCK_D, + CONCAT_UX=concat_ux, + SILU_U=silu_u, + TRAINING=training, + num_warps=1, + ) + + return y, mean, rstd, BLOCK_D, 1, seed # pyre-ignore [7] + + +@triton.jit +def _helion_ln_mul_dropout_bwd_dx_du( + DX, + DU, + DY, + DW, + DB, + X, + U, + Y, + W, + B, + Mean, + Rstd, + stride_dx, + stride_du, + stride_dy, + stride_x, + stride_u, + stride_y, + D: tl.constexpr, + eps, + seed, + dropout_ratio, + N, + SILU_U: tl.constexpr, + BLOCK_D: tl.constexpr, + TRAINING: tl.constexpr, + CONCAT_UX: tl.constexpr, + COMPUTE_Y: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + rows_per_tile = N // tile_num + if pid < N % tile_num: + rows_per_tile += 1 + + if rows_per_tile == 0: + return + + cols = tl.arange(0, BLOCK_D) + mask = cols < D + + # precompute inverse of D + inv_D: tl.constexpr = 1.0 / D + + row = pid + X += row.to(tl.int64) * stride_x + U += row.to(tl.int64) * stride_u + if COMPUTE_Y: + Y += row.to(tl.int64) * stride_y + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + DU += row.to(tl.int64) * stride_du + DW = DW + pid * D + cols + DB = DB + pid * D + cols + + partial_dw = tl.zeros((BLOCK_D,), dtype=tl.float32) + partial_db = tl.zeros((BLOCK_D,), dtype=tl.float32) + w = tl.load(W + cols, mask=mask).to(tl.float32) + b = tl.load(B + cols, mask=mask).to(tl.float32) + + for _idx in range(0, rows_per_tile): + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + if CONCAT_UX: + du = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + dx = tl.load(DY + D + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + 2 * D + cols, mask=mask, other=0).to(tl.float32) + else: + du = tl.zeros([BLOCK_D], dtype=tl.float32) + dx = tl.zeros([BLOCK_D], dtype=tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + + if TRAINING: + # pyre-fixme[16] + dropout_scale = fast_dividef(1.0, 1.0 - dropout_ratio) + if CONCAT_UX: + random_offsets = 3 * row * BLOCK_D + cols + # apply dropout on du + random_du, random_dx, random_dy = rand3x(seed, random_offsets) + du_keep = random_du > dropout_ratio + du = tl.where(du_keep, du * dropout_scale, 0.0) + # apply dropout on dx + dx_keep = random_dx > dropout_ratio + dx = tl.where(dx_keep, dx * dropout_scale, 0.0) + # apply dropout on dy + dy_keep = random_dy > dropout_ratio + dy = tl.where(dy_keep, dy * dropout_scale, 0.0) + else: + random_offsets = row * BLOCK_D + cols + random = tl.rand(seed, random_offsets) + dy_keep = random > dropout_ratio + dy = tl.where(dy_keep, dy * dropout_scale, 0.0) + + mean = tl.load(Mean + row) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = (x - mean) * rstd + u = tl.load(U + cols, mask=mask, other=0).to(tl.float32) + ln = xhat * w + b + du += dy * ln + + if SILU_U: + sig_u = tl.sigmoid(u) + silu_u = u * sig_u + du = du * sig_u * (1 + u - silu_u) + u = silu_u + + tl.store(DU + cols, du.to(DU.dtype.element_ty), mask=mask) + dy = dy * u + wdy = w * dy + + if COMPUTE_Y: + y = ln * u + if TRAINING: + # pyre-fixme[16] + dropout_scale_y = fast_dividef(1.0, 1.0 - dropout_ratio) + if CONCAT_UX: + u = tl.where(du_keep, u * dropout_scale_y, 0.0) # pyre-ignore [61] + x = tl.where(dx_keep, x * dropout_scale_y, 0.0) # pyre-ignore [61] + y = tl.where(dy_keep, y * dropout_scale_y, 0.0) # pyre-ignore [61] + else: + y = tl.where(dy_keep, y * dropout_scale_y, 0.0) # pyre-ignore [61] + if CONCAT_UX: + tl.store(Y + cols, u.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + D + cols, x.to(Y.dtype.element_ty), mask=mask) + tl.store(Y + 2 * D + cols, y.to(Y.dtype.element_ty), mask=mask) + else: + tl.store(Y + cols, y.to(Y.dtype.element_ty), mask=mask) + Y += tile_num.to(tl.int64) * stride_y + + xhat = tl.where(mask, xhat, 0.0) + wdy = tl.where(mask, wdy, 0.0) + # multiply by inv_D + c1 = tl.sum(xhat * wdy, axis=0) * inv_D + c2 = tl.sum(wdy, axis=0) * inv_D + dx += (wdy - (xhat * c1 + c2)) * rstd + + # Write dx + tl.store(DX + cols, dx, mask=mask) + + # Accumulate partial sums for dw/db + partial_dw += dy * xhat + partial_db += dy + + X += tile_num.to(tl.int64) * stride_x + U += tile_num.to(tl.int64) * stride_u + DY += tile_num.to(tl.int64) * stride_dy + DX += tile_num.to(tl.int64) * stride_dx + DU += tile_num.to(tl.int64) * stride_du + row += tile_num + + tl.store(DW, partial_dw, mask=mask) + tl.store(DB, partial_db, mask=mask) + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=["D"], +) +@triton.jit +def _helion_ln_mul_dropout_bwd_dwdb( + DW, + DB, + FINAL_DW, + FINAL_DB, + N, + D, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid = tl.program_id(0) + cols = pid * BLOCK_D + tl.arange(0, BLOCK_D) + dw = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + db = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + off_mask = (rows[:, None] < N) & (cols[None, :] < D) + offs = rows[:, None] * D + cols[None, :] + dw += tl.load(DW + offs, mask=off_mask, other=0.0) + db += tl.load(DB + offs, mask=off_mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + sum_db = tl.sum(db, axis=0) + tl.store(FINAL_DW + cols, sum_dw.to(FINAL_DW.dtype.element_ty), mask=cols < D) + tl.store(FINAL_DB + cols, sum_db.to(FINAL_DB.dtype.element_ty), mask=cols < D) + + +def helion_layer_norm_mul_dropout_bwd( + dy: torch.Tensor, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + BLOCK_D: int, + num_warps: int, + eps: float, + training: bool, + dropout_ratio: float, + seed: Optional[int] = None, + silu_u: bool = False, + concat_ux: bool = False, + compute_y: bool = False, +) -> Tuple[ + torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, Optional[torch.Tensor] +]: + y = None + N, D = x.shape + if compute_y: + if concat_ux: + y = torch.empty((N, 3 * D), dtype=x.dtype, device=x.device) + else: + y = torch.empty_like(x) + if N == 0: + return ( + torch.zeros_like(x), + torch.zeros_like(u), + torch.zeros((D,), dtype=weight.dtype, device=x.device), + torch.zeros((D,), dtype=weight.dtype, device=x.device), + y, + ) + dx = torch.empty_like(x) + du = torch.empty_like(u) + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + tile_num = max(1, min(sms * 64, N // 4)) + _dweight = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + _dbias = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + + # pyre-ignore[28] + _helion_ln_mul_dropout_bwd_dx_du[(tile_num,)]( + dx, + du, + dy, + _dweight, + _dbias, + x, + u, + y, + weight, + bias, + mean, + rstd, + dx.stride(0), + du.stride(0), + dy.stride(0), + x.stride(0), + u.stride(0), + y.stride(0) if compute_y else 0, # pyre-ignore [16] + D, + eps, + seed, + dropout_ratio, + N=N, + SILU_U=silu_u, + BLOCK_D=BLOCK_D, + TRAINING=training, + CONCAT_UX=concat_ux, + COMPUTE_Y=compute_y, + num_warps=num_warps, + ) + + blocks = triton.next_power_of_2(sms * 4) + BLOCK_D_DWDB = triton.next_power_of_2(triton.cdiv(D, blocks)) + BLOCK_D_DWDB = min(max(BLOCK_D_DWDB, 4), 128) + _helion_ln_mul_dropout_bwd_dwdb[(triton.cdiv(D, BLOCK_D_DWDB),)]( + _dweight, + _dbias, + dweight, + dbias, + tile_num, + D, + BLOCK_D=BLOCK_D_DWDB, + ) + return dx, du, dweight, dbias, y + + +class HelionLayerNormMulDropoutFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + seed: Optional[int] = None, + ) -> torch.Tensor: + if dropout_ratio == 0.0: + # skip dropout computation if dropout ratio is 0 + training = False + y, mean, rstd, BLOCK_D, num_warps, seed = helion_layer_norm_mul_dropout_fwd( + x=x, + u=u, + weight=weight, + bias=bias, + eps=eps, + dropout_ratio=dropout_ratio, + training=training, + silu_u=silu_u, + concat_ux=concat_ux, + seed=seed, + ) + ctx.save_for_backward(x, u, weight, bias, mean, rstd) + ctx.BLOCK_D = BLOCK_D + ctx.num_warps = num_warps + ctx.eps = eps + ctx.seed = seed + ctx.training = training + ctx.silu_u = silu_u + ctx.concat_ux = concat_ux + ctx.dropout_ratio = dropout_ratio + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[ + torch.Tensor, + torch.Tensor, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + ]: + x, u, weight, bias, mean, rstd = ctx.saved_tensors + dx, du, dweight, dbias, _ = helion_layer_norm_mul_dropout_bwd( + dy=dy, + x=x, + u=u, + weight=weight, + bias=bias, + mean=mean, + rstd=rstd, + BLOCK_D=ctx.BLOCK_D, + num_warps=ctx.num_warps, + eps=ctx.eps, + training=ctx.training, + dropout_ratio=ctx.dropout_ratio, + seed=ctx.seed, + silu_u=ctx.silu_u, + concat_ux=ctx.concat_ux, + compute_y=False, + ) + return dx, du, dweight, dbias, None, None, None, None, None, None + + +@torch.fx.wrap +def helion_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_ux: bool = False, + seed: Optional[int] = None, +) -> torch.Tensor: + return HelionLayerNormMulDropoutFunction.apply( + x, u, weight, bias, eps, dropout_ratio, training, silu_u, concat_ux, seed + ) + + +@torch.fx.wrap +def triton_norm_mul_dropout( + x: torch.Tensor, + u: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, +) -> torch.Tensor: + if group_norm: + return GroupNormMulDropoutFunction.apply( + x, + u, + weight, + bias, + eps, + dropout_ratio, + training, + silu_u, + concat_u and concat_x, + num_heads, + linear_dim, + seed, + ) + else: + return LayerNormMulDropoutFunction.apply( + x, + u, + weight, + bias, + eps, + dropout_ratio, + training, + silu_u, + concat_u and concat_x, + seed, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_hstu_compute_output( + attn: torch.Tensor, + u: torch.Tensor, + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + output_weight: torch.Tensor, + eps: float, + dropout_ratio: float, + training: bool, + silu_u: bool = False, + concat_u: bool = False, + concat_x: bool = False, + mul_u_activation_type: str = "none", + group_norm: bool = False, + num_heads: int = 1, + linear_dim: int = -1, + seed: Optional[int] = None, + recompute_y_in_backward: bool = False, +) -> torch.Tensor: + return HSTUComputeOutputFunction.apply( + attn, + u, + x, + norm_weight, + norm_bias, + output_weight, + eps, + dropout_ratio, + training, + silu_u, + concat_u, + concat_x, + mul_u_activation_type, + group_norm, + num_heads, + linear_dim, + seed, + recompute_y_in_backward, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_preprocess_and_attention.py b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_preprocess_and_attention.py new file mode 100644 index 000000000..bda97ff96 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_hstu_preprocess_and_attention.py @@ -0,0 +1,342 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +from typing import Optional, Tuple + +import torch +from generative_recommenders.ops.triton.triton_addmm import ( + maybe_triton_addmm_fwd, + triton_addmm_bwd, + triton_addmm_fwd, +) +from generative_recommenders.ops.triton.triton_hstu_attention import ( + _should_enable_tma, + triton_hstu_attention_bwd, + triton_hstu_attention_fwd, +) +from generative_recommenders.ops.triton.triton_layer_norm import ( + compute_BLOCK_D, + triton_weighted_layer_norm_bwd, + triton_weighted_layer_norm_fwd, +) +from torch.nn import functional as F + + +class _HSTUPreprocessAndAttentionFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore [14] + def forward( + ctx, # pyre-ignore [2] + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: torch.Tensor, + max_seq_len: int, + seq_offsets: torch.Tensor, + attn_alpha: float, + num_targets: Optional[torch.Tensor], + max_attn_len: int, + contextual_seq_len: int, + recompute_uvqk_in_backward: bool, + recompute_normed_x_in_backward: bool, + sort_by_length: bool, + enable_tma: bool, + num_softmax_heads: int = 0, + ) -> Tuple[torch.Tensor, torch.Tensor]: + assert num_softmax_heads == 0, "Softmax attention is not supported" + normed_x, x_mean, x_rstd = triton_weighted_layer_norm_fwd( + x=x, + weight=norm_weight, + bias=norm_bias, + eps=norm_eps, + ) + BLOCK_D = compute_BLOCK_D(x) + uvqk = maybe_triton_addmm_fwd( + x=normed_x, w=uvqk_weight, y=uvqk_bias + ).contiguous() + u, v, q, k = uvqk.split( + [ + hidden_dim * num_heads, + hidden_dim * num_heads, + attn_dim * num_heads, + attn_dim * num_heads, + ], + dim=1, + ) + q = q.view(-1, num_heads, attn_dim) + k = k.view(-1, num_heads, attn_dim) + v = v.view(-1, num_heads, hidden_dim) + silu_u = F.silu(u) + sort_by_length_indices = None + if sort_by_length: + seq_lengths = seq_offsets[1:] - seq_offsets[:-1] + _, sort_by_length_indices = torch.sort( + seq_lengths, descending=True, stable=False + ) + out = triton_hstu_attention_fwd( + N=max_seq_len, + alpha=attn_alpha, + q=q, + k=k, + v=v, + seq_offsets=seq_offsets, + num_targets=num_targets, + max_attn_len=max_attn_len, + contextual_seq_len=contextual_seq_len, + sort_by_length_indices=sort_by_length_indices, + enable_tma=enable_tma, + num_softmax_heads=num_softmax_heads, + ) + # update ctx + saved_tensors = [ + x, + norm_weight, + norm_bias, + x_mean, + x_rstd, + uvqk_weight, + seq_offsets, + ] + if num_targets is not None: + saved_tensors.append(num_targets) + if not recompute_normed_x_in_backward: + saved_tensors.append(normed_x) + if recompute_uvqk_in_backward: + saved_tensors.append(uvqk_bias) + else: + saved_tensors.append(uvqk) + if sort_by_length: + saved_tensors.append(sort_by_length_indices) + ctx.save_for_backward(*saved_tensors) + ctx.attn_alpha = attn_alpha + ctx.has_multiple_targets = num_targets is not None + ctx.max_seq_len = max_seq_len + ctx.max_attn_len = max_attn_len + ctx.recompute_normed_x_in_backward = recompute_normed_x_in_backward + ctx.recompute_uvqk_in_backward = recompute_uvqk_in_backward + ctx.hidden_dim = hidden_dim + ctx.attn_dim = attn_dim + ctx.num_heads = num_heads + ctx.uvqk_bias_1d = uvqk_bias.dim() == 1 + ctx.norm_eps = norm_eps + ctx.norm_BLOCK_D = BLOCK_D + ctx.contextual_seq_len = contextual_seq_len + ctx.sort_by_length = sort_by_length + ctx.enable_tma = enable_tma + ctx.num_softmax_heads = num_softmax_heads + return silu_u, out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, # pyre-ignore[2] + dsilu_u: torch.Tensor, + dout: torch.Tensor, + ) -> Tuple[ + torch.Tensor, # d_x + torch.Tensor, # d_norm_weight + torch.Tensor, # d_norm_bias + None, + None, + None, + None, + torch.Tensor, # d_uvqk_weight + torch.Tensor, # d_uvqk_bias + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ]: + x, norm_weight, norm_bias, x_mean, x_rstd, uvqk_weight, seq_offsets = ( + ctx.saved_tensors[:7] + ) + idx = 7 + if ctx.has_multiple_targets: + num_targets = ctx.saved_tensors[idx] + idx += 1 + else: + num_targets = None + if ctx.recompute_normed_x_in_backward: + normed_x, _, _ = triton_weighted_layer_norm_fwd( + x=x, + weight=norm_weight, + bias=norm_bias, + eps=ctx.norm_eps, + mean=x_mean, + rstd=x_rstd, + ) + else: + normed_x = ctx.saved_tensors[idx] + idx += 1 + if ctx.recompute_uvqk_in_backward: + uvqk_bias = ctx.saved_tensors[idx] + uvqk = maybe_triton_addmm_fwd(x=normed_x, w=uvqk_weight, y=uvqk_bias) + idx += 1 + else: + uvqk = ctx.saved_tensors[idx] + idx += 1 + if ctx.sort_by_length: + sort_by_length_indices = ctx.saved_tensors[idx] + else: + sort_by_length_indices = None + + duvqk = torch.empty_like(uvqk) + du, dv, dq, dk = duvqk.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + u, v, q, k = uvqk.split( + [ + ctx.hidden_dim * ctx.num_heads, + ctx.hidden_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ctx.attn_dim * ctx.num_heads, + ], + dim=1, + ) + q = q.view(-1, ctx.num_heads, ctx.attn_dim) + k = k.view(-1, ctx.num_heads, ctx.attn_dim) + v = v.view(-1, ctx.num_heads, ctx.hidden_dim) + dq = dq.view(-1, ctx.num_heads, ctx.attn_dim) + dk = dk.view(-1, ctx.num_heads, ctx.attn_dim) + dv = dv.view(-1, ctx.num_heads, ctx.hidden_dim) + # Note: the operation below updates duvqk in place + triton_hstu_attention_bwd( + dout=dout, + q=q, + k=k, + v=v, + dq=dq, + dk=dk, + dv=dv, + seq_offsets=seq_offsets, + num_targets=num_targets, + N=ctx.max_seq_len, + max_attn_len=ctx.max_attn_len, + alpha=ctx.attn_alpha, + contextual_seq_len=ctx.contextual_seq_len, + sort_by_length_indices=sort_by_length_indices, + enable_tma=ctx.enable_tma, + num_softmax_heads=ctx.num_softmax_heads, + ) + torch.ops.aten.silu_backward(dsilu_u, u, grad_input=du) + d_normed_x, d_uvqk_weight, d_uvqk_bias = triton_addmm_bwd( + x=normed_x, + w=uvqk_weight, + dz=duvqk, + is_y_1d=ctx.uvqk_bias_1d, + ) + d_x, d_norm_weight, d_norm_bias = triton_weighted_layer_norm_bwd( + dy=d_normed_x, + x=x, + weight=norm_weight, + bias=norm_bias, + mean=x_mean, + rstd=x_rstd, + learnable=True, + eps=ctx.norm_eps, + BLOCK_D=ctx.norm_BLOCK_D, + ) + # pyre-ignore[7] + return ( + d_x, + d_norm_weight, + d_norm_bias, + None, + None, + None, + None, + d_uvqk_weight, + d_uvqk_bias, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + ) + + +def triton_hstu_preprocess_and_attention( + x: torch.Tensor, + norm_weight: torch.Tensor, + norm_bias: torch.Tensor, + norm_eps: float, + num_heads: int, + attn_dim: int, + hidden_dim: int, + uvqk_weight: torch.Tensor, + uvqk_bias: torch.Tensor, + max_seq_len: int, + seq_offsets: torch.Tensor, + attn_alpha: float, + num_targets: Optional[torch.Tensor], + max_attn_len: int = 0, + contextual_seq_len: int = 0, + recompute_uvqk_in_backward: bool = False, + recompute_normed_x_in_backward: bool = False, + sort_by_length: bool = False, + enable_tma: Optional[bool] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + # When the caller does not specify enable_tma, auto-detect whether the + # TMA / TLX fast path is safe on this device. Resolving here (vs inside + # the autograd Function.forward) keeps a concrete bool flowing through + # ctx.save_for_backward / ctx attributes. + if enable_tma is None: + enable_tma = _should_enable_tma() + return _HSTUPreprocessAndAttentionFunction.apply( + x, + norm_weight, + norm_bias, + norm_eps, + num_heads, + attn_dim, + hidden_dim, + uvqk_weight, + uvqk_bias, + max_seq_len, + seq_offsets, + attn_alpha, + num_targets, + max_attn_len, + contextual_seq_len, + recompute_uvqk_in_backward, + recompute_normed_x_in_backward, + sort_by_length, + enable_tma, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py new file mode 100644 index 000000000..3f5609d75 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged.py @@ -0,0 +1,2537 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +import os +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + autotune_max_seq_len, + fine_grained_autotune_max_seq_len, + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.triton._autotune_pinning import pinned_or_full +from generative_recommenders.ops.utils import is_sm100_plus, is_sm90 +from torch._inductor.runtime import triton_helpers + +try: + torch.ops.load_library( + "//generative_recommenders/fb/ultra/ops/hopper/jagged_dense_bmm_add:jagged_dense_bmm_add" + ) +except OSError: + pass + +CUDA_JAGGED_DENSE_BMM_FWD = False +CUDA_JAGGED_DENSE_BMM_BWD = False + +SPLIT_2D_JAGGED_KERNEL = None +GLN_MUL_DROPOUT_KERNEL = None +CONCAT_2D_JAGGED_KERNEL = None + + +def set_cuda_jagged_dense_bmm_fwd(value: bool) -> None: + global CUDA_JAGGED_DENSE_BMM_FWD + CUDA_JAGGED_DENSE_BMM_FWD = value + + +def get_cuda_jagged_dense_bmm_fwd() -> bool: + # currently only supports H100 + return CUDA_JAGGED_DENSE_BMM_FWD and is_sm90() + + +def set_cuda_jagged_dense_bmm_bwd(value: bool) -> None: + global CUDA_JAGGED_DENSE_BMM_BWD + CUDA_JAGGED_DENSE_BMM_BWD = value + + +def get_cuda_jagged_dense_bmm_bwd() -> bool: + # currently only supports H100 + return CUDA_JAGGED_DENSE_BMM_BWD and is_sm90() + + +def set_split_2d_jagged_kernel(value: Optional[str]) -> None: + global SPLIT_2D_JAGGED_KERNEL + SPLIT_2D_JAGGED_KERNEL = value + + +def get_split_2d_jagged_kernel() -> Optional[str]: + # only override during training + if torch.is_grad_enabled(): + return SPLIT_2D_JAGGED_KERNEL + return None + + +def set_concat_2d_jagged_kernel(value: Optional[str]) -> None: + global CONCAT_2D_JAGGED_KERNEL + CONCAT_2D_JAGGED_KERNEL = value + + +def get_concat_2d_jagged_kernel() -> Optional[str]: + # only override during training + if torch.is_grad_enabled(): + return CONCAT_2D_JAGGED_KERNEL + return None + + +def _should_use_multirow() -> bool: + """Check if multirow kernel should be used based on current hardware. + + Can be overridden via the JAGGED_USE_MULTIROW_MI350 environment variable: + JAGGED_USE_MULTIROW_MI350=1 -> force multirow on + JAGGED_USE_MULTIROW_MI350=0 -> force multirow off + unset -> auto-detect based on hardware (SM100+ or MI350) + """ + env = os.environ.get("JAGGED_USE_MULTIROW_MI350") + if env is not None: + return env == "1" + return is_sm100_plus() + + +def set_gln_mul_dropout_kernel(value: Optional[str]) -> None: + global GLN_MUL_DROPOUT_KERNEL + GLN_MUL_DROPOUT_KERNEL = value + + +def get_gln_mul_dropout_kernel() -> Optional[str]: + # only override during training + return GLN_MUL_DROPOUT_KERNEL + + +def _triton_concat_2D_jagged_internal( + values_a: torch.Tensor, + values_b: torch.Tensor, + values_out: torch.Tensor, + max_seq_len: int, + B: int, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + D: int, + dense_size: int, + stride_dense_batch: int, + n_prefix: int, + is_dense_a: bool, + is_dense_b: bool, + is_replace: bool, + BLOCK_D: int, +) -> None: + use_multirow = _should_use_multirow() + if n_prefix != 0: + if use_multirow: + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + concat_2D_jagged_jagged_w_prefix_multirow[grid]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_od=values_out.stride(0), + n_prefix_from_B=n_prefix, + BLOCK_D=BLOCK_D, + ) + else: + concat_2D_jagged_jagged_w_prefix[(max_seq_len, B)]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_od=values_out.stride(0), + n_prefix_from_B=n_prefix, + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + else: + if use_multirow: + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + concat_2D_jagged_multirow[grid]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + DenseSize=dense_size, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_dense_batch=stride_dense_batch, + stride_od=values_out.stride(0), + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + IS_REPLACE=is_replace, # pyre-ignore[6] + ) + else: + concat_2D_jagged[(max_seq_len, B)]( + OffsetsA=offsets_a, + ValuesA=values_a, + OffsetsB=offsets_b, + ValuesB=values_b, + DenseSize=dense_size, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_dense_batch=stride_dense_batch, + stride_od=values_out.stride(0), + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + IS_REPLACE=is_replace, # pyre-ignore[6] + ) + + +def _get_split_concat_2d_jagged_multirow_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [1, 2, 4, 8]: + for num_warps in [1, 2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +def _get_split_concat_2d_jagged_multirow_configs_wrapper() -> List[triton.Config]: + # Use extended config space only when JAGGED_USE_MULTIROW_MI350 is explicitly set, + # otherwise fall back to the existing configs to avoid breaking autotune. + if os.environ.get("JAGGED_USE_MULTIROW_MI350") is not None: + configs = [] + # Extended config space for MI350 tuning + # - BLOCK_N: number of rows processed per block + # - num_warps: number of warps (AMD wavefront = 64 threads) + # - num_stages: software pipeline depth for memory latency hiding + # NOTE: num_stages=0 is invalid for AMD GPUs, start from 1 + # - waves_per_eu: AMD-specific, controls occupancy (waves per execution unit) + for BLOCK_N in [1, 2, 4, 8, 16, 32]: + for num_warps in [1, 2, 4, 8, 16, 32]: + for num_stages in [1, 2, 3, 4]: + for waves_per_eu in [0, 1, 2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N, "waves_per_eu": waves_per_eu}, + num_warps=num_warps, + num_stages=num_stages, + ) + ) + return configs + return _get_split_concat_2d_jagged_multirow_configs() + + +def _get_bmm_configs() -> List[triton.Config]: + configs = [] + for BLOCK_M in [64, 128]: + for BLOCK_N in [64, 128, 256]: + for BLOCK_K in [32, 64]: + for num_stages in [3, 5]: + for num_warps in [4, 8]: + configs.append( + triton.Config( + { + "BLOCK_M": BLOCK_M, + "BLOCK_N": BLOCK_N, + "BLOCK_K": BLOCK_K, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bmm_configs(), + key=["AUTOTUNE_MAX_SEQ_LEN", "N", "K", "ELEMENTWISE", "HAS_BIAS"], +) +@triton.jit +def jagged_dense_bmm_broadcast_add_kernel( + seq_offsets, + Jagged, + Dense, + Bias, + Out, + AUTOTUNE_MAX_SEQ_LEN, + N, + K, + stride_jm, + stride_db, + stride_dk, + stride_dn, + stride_bias_b, + stride_om, + HAS_BIAS: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + ELEMENTWISE: tl.constexpr, +): + """ + Computing bmm Out = Jagged x Dense + Bias + M is the jagged dimension + Jagged has shape (sum_B(M_i), K), Dense has shape (B, K, N), Bias has shape (B, N), and Out has shape (sum_B(M_i), N) + """ + + off_n = tl.program_id(0) + off_m = tl.program_id(1).to(tl.int64) + off_b = tl.program_id(2) + + seq_start = tl.load(seq_offsets + off_b).to(tl.int64) + seq_end = tl.load(seq_offsets + off_b + 1) + seq_len = seq_end - seq_start + start_m = off_m * BLOCK_M + start_n = off_n * BLOCK_N + if start_m >= seq_len: + return + + Jagged += (seq_start + start_m) * stride_jm + Dense += off_b.to(tl.int64) * stride_db + Out += seq_start * stride_om + + offs_m = tl.arange(0, BLOCK_M) + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_k = tl.arange(0, BLOCK_K) + jg_ptrs = Jagged + offs_m[:, None] * stride_jm + offs_k[None, :] + dn_ptrs = Dense + offs_k[:, None] * stride_dk + offs_n[None, :] * stride_dn + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + for k in range(0, K, BLOCK_K): + jg = tl.load( + jg_ptrs, + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask=(offs_m[:, None] < (seq_len - start_m)) & ((k + offs_k)[None, :] < K), + other=0.0, + ) + dn = tl.load( + dn_ptrs, + mask=((k + offs_k)[:, None] < K) & (offs_n[None, :] < N), + other=0.0, + ) + accumulator += tl.dot(jg, dn, allow_tf32=ALLOW_TF32) + jg_ptrs += BLOCK_K + dn_ptrs += BLOCK_K * stride_dk + + if HAS_BIAS: + if ELEMENTWISE: + Bias += (seq_start + start_m) * stride_bias_b + bias_ptrs = Bias + offs_m[:, None] * stride_bias_b + offs_n[None, :] + bias = tl.load( + bias_ptrs, + mask=(offs_m[:, None] < (seq_len - start_m)) & (offs_n[None, :] < N), + other=0.0, + ) + accumulator += bias.to(tl.float32) + else: + bias_ptrs = Bias + off_b.to(tl.int64) * stride_bias_b + offs_n + bias = tl.load(bias_ptrs, mask=offs_n < N) + accumulator += bias[None, :].to(tl.float32) + + out = accumulator.to(Out.dtype.element_ty) + + offs_m = tl.arange(0, BLOCK_M) + offs_n = start_n + tl.arange(0, BLOCK_N) + Out += start_m * stride_om + out_ptrs = Out + offs_m[:, None] * stride_om + offs_n[None, :] + tl.store( + out_ptrs, + out, + mask=(offs_m[:, None] < (seq_len - start_m)) & (offs_n[None, :] < N), + ) + + +def _get_bmm_reduce_sum_configs() -> List[triton.Config]: + configs = [] + for BLOCK_M in [64, 128]: + for BLOCK_N in [64, 128]: + for BLOCK_K in [64, 128]: + for num_stages in [3, 4]: + for num_warps in [4, 8]: + configs.append( + triton.Config( + { + "BLOCK_M": BLOCK_M, + "BLOCK_N": BLOCK_N, + "BLOCK_K": BLOCK_K, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_bmm_reduce_sum_configs(), + key=["M", "N", "AUTOTUNE_MAX_SEQ_LEN"], +) +@triton.jit +def _jagged_jagged_bmm_reduce_sum( + seq_offsets, + JaggedA, + JaggedB, + Out, + ReduceOut, + M, + N, + AUTOTUNE_MAX_SEQ_LEN, + stride_ak, + stride_bk, + stride_ob, + stride_om, + stride_on, + stride_orb, + stride_orn, + REDUCE_JAGGEDB: tl.constexpr, + ALLOW_TF32: tl.constexpr, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, +): + """ + Computing bmm Out = Jagged x Jagged + K is the jagged dimension + JaggedA has shape (sum_B(K_i), M), JaggedB has shape (sum_B(K_i), N), and Out has shape (B, M, N) + """ + + off_m = tl.program_id(0).to(tl.int64) + off_n = tl.program_id(1) + off_b = tl.program_id(2) + + seq_start = tl.load(seq_offsets + off_b).to(tl.int64) + seq_end = tl.load(seq_offsets + off_b + 1) + seq_len = seq_end - seq_start + + start_m = off_m * BLOCK_M + start_n = off_n * BLOCK_N + + accumulator = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + Out += off_b.to(tl.int64) * stride_ob + offs_m = tl.arange(0, BLOCK_M) + offs_n = start_n + tl.arange(0, BLOCK_N) + Out += start_m * stride_om + out_ptrs = Out + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on + if REDUCE_JAGGEDB: + out_reduce_ptrs = ( + ReduceOut + off_b.to(tl.int64) * stride_orb + offs_n * stride_orn + ) + acc_reduce = tl.zeros((BLOCK_N,), dtype=tl.float32) + if seq_len == 0: + out = accumulator.to(Out.dtype.element_ty) + tl.store( + out_ptrs, + out, + mask=(offs_m[:, None] < (M - start_m)) & (offs_n[None, :] < N), + ) + if REDUCE_JAGGEDB: + if off_m == 0: + tl.store( + out_reduce_ptrs, # pyre-ignore [61] + acc_reduce.to(ReduceOut.dtype.element_ty), + mask=(offs_n < N), + ) + return + + JaggedA += seq_start * stride_ak + JaggedB += seq_start * stride_bk + offs_k = tl.arange(0, BLOCK_K) + jg_a_ptrs = JaggedA + offs_k[None, :] * stride_ak + (start_m + offs_m)[:, None] + jg_b_ptrs = JaggedB + offs_k[:, None] * stride_bk + offs_n[None, :] + + for k in range(0, seq_len, BLOCK_K): + jg_a = tl.load( + jg_a_ptrs, + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask=(offs_m[:, None] < (M - start_m)) & ((k + offs_k)[None, :] < seq_len), + other=0.0, + ) + jg_b = tl.load( + jg_b_ptrs, + mask=(offs_n[None, :] < N) & ((k + offs_k)[:, None] < seq_len), + other=0.0, + ) + + accumulator += tl.dot(jg_a, jg_b, allow_tf32=ALLOW_TF32) + if REDUCE_JAGGEDB: + if off_m == 0: + acc_reduce += tl.sum(jg_b.to(tl.float32), axis=0) + + jg_a_ptrs += BLOCK_K * stride_ak + jg_b_ptrs += BLOCK_K * stride_bk + + out = accumulator.to(Out.dtype.element_ty) + tl.store( + out_ptrs, + out, + mask=(offs_m[:, None] < (M - start_m)) & (offs_n[None, :] < N), + ) + if REDUCE_JAGGEDB: + if off_m == 0: + tl.store( + out_reduce_ptrs, # pyre-ignore [61] + acc_reduce.to(ReduceOut.dtype.element_ty), + mask=(offs_n < N), + ) + + +class _JaggedDenseBmmFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + ): + jagged = switch_to_contiguous_if_needed(jagged) + L, D = jagged.shape + B, _, K = dense.shape + bmm_out = torch.empty((L, K), dtype=jagged.dtype, device=jagged.device) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(K, meta["BLOCK_N"]), + triton.cdiv(max_seq_len, meta["BLOCK_M"]), + B, + ) + + jagged_dense_bmm_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=jagged, + Dense=dense, + Bias=0, + Out=bmm_out, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + N=K, + K=D, + stride_jm=jagged.stride(0), + stride_db=dense.stride(0), + stride_dk=dense.stride(1), + stride_dn=dense.stride(2), + stride_bias_b=0, + stride_om=bmm_out.stride(0), + HAS_BIAS=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ELEMENTWISE=False, + ) + + ctx.save_for_backward(seq_offsets, jagged, dense) + ctx.B = B + ctx.max_seq_len = max_seq_len + ctx.K = K + ctx.D = D + return bmm_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_bmm_out: torch.Tensor + ) -> Tuple[None, None, torch.Tensor, torch.Tensor]: + seq_offsets, jagged, dense = ctx.saved_tensors + d_jagged = torch.empty_like(jagged) + d_dense = torch.empty_like(dense) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(ctx.D, meta["BLOCK_N"]), + triton.cdiv(ctx.max_seq_len, meta["BLOCK_M"]), + ctx.B, + ) + jagged_dense_bmm_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=d_bmm_out, + Dense=dense, + Bias=None, + Out=d_jagged, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(ctx.max_seq_len), + N=ctx.D, + K=ctx.K, + stride_jm=d_bmm_out.stride(0), + stride_db=dense.stride(0), + stride_dk=dense.stride(2), + stride_dn=dense.stride(1), + stride_bias_b=0, + stride_om=d_jagged.stride(0), + HAS_BIAS=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ELEMENTWISE=False, + ) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(ctx.D, meta["BLOCK_M"]), + triton.cdiv(ctx.K, meta["BLOCK_N"]), + ctx.B, + ) + _jagged_jagged_bmm_reduce_sum[grid]( + seq_offsets=seq_offsets, + JaggedA=jagged, + JaggedB=d_bmm_out, + Out=d_dense, + ReduceOut=None, + M=ctx.D, + N=ctx.K, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(ctx.max_seq_len), + stride_ak=jagged.stride(0), + stride_bk=d_bmm_out.stride(0), + stride_ob=d_dense.stride(0), + stride_om=d_dense.stride(1), + stride_on=d_dense.stride(2), + stride_orb=0, + stride_orn=0, + REDUCE_JAGGEDB=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ) + + return None, None, d_jagged, d_dense + + +def _get_jagged_dense_broadcast_add_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [16, 32, 64]: + for num_stages in [1, 2]: + for num_warps in [2, 4, 8]: + configs.append( + triton.Config( + { + "BLOCK_N": BLOCK_N, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_jagged_dense_broadcast_add_configs(), + key=["AUTOTUNE_MAX_SEQ_LEN"], +) +@triton.jit +def jagged_dense_broadcast_add_kernel( + seq_offsets, + Jagged, + Dense, + Out, + AUTOTUNE_MAX_SEQ_LEN, + D, + stride_jn, + stride_db, + stride_on, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + """ + Computing Out = Jagged + Dense + JaggedA has shape (sum_B(N_i), D), Dense has shape (B, D), and Out has shape (sum_B(N_i), D) + """ + + off_b = tl.program_id(0) + off_n = tl.program_id(1) + seq_start = tl.load(seq_offsets + off_b) + seq_end = tl.load(seq_offsets + off_b + 1) + seq_len = seq_end - seq_start + start_n = off_n * BLOCK_N + if start_n >= seq_len: + return + Jagged += seq_start * stride_jn + Dense += off_b * stride_db + Out += seq_start * stride_on + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + jagged_ptrs = Jagged + offs_n[:, None] * stride_jn + offs_d[None, :] + dense_ptrs = Dense + offs_d + out_ptrs = Out + offs_n[:, None] * stride_jn + offs_d[None, :] + for d in range(0, D, BLOCK_D): + jg = tl.load( + jagged_ptrs, + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask=(offs_n[:, None] < seq_len) & ((d + offs_d)[None, :] < D), + ) + dn = tl.load(dense_ptrs, mask=d + offs_d < D) + out = jg + dn[None, :] + tl.store( + out_ptrs, + out, + mask=(offs_n[:, None] < seq_len) & ((d + offs_d)[None, :] < D), + ) + dense_ptrs += BLOCK_D + jagged_ptrs += BLOCK_D + out_ptrs += BLOCK_D + + +@triton.jit +def jagged_reduce_sum( + seq_offsets, + Jagged, + Out, + D, + stride_jn, + stride_ob, + BLOCK_D: tl.constexpr, +): + """ + Computing Out = Jagged + Dense + JaggedA has shape (sum_B(N_i), D), Dense has shape (B, D), and Out has shape (sum_B(N_i), D) + """ + off_b = tl.program_id(0) + off_d = tl.program_id(1) * BLOCK_D + seq_start = tl.load(seq_offsets + off_b) + seq_end = tl.load(seq_offsets + off_b + 1) + seq_len = seq_end - seq_start + Jagged += seq_start * stride_jn + Out += off_b * stride_ob + offs_d = off_d + tl.arange(0, BLOCK_D) + jagged_ptrs = Jagged + offs_d + out_ptrs = Out + offs_d + accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32) + for _ in range(0, seq_len): + jg = tl.load( + jagged_ptrs, + mask=offs_d < D, + ) + accumulator += jg + jagged_ptrs += stride_jn + out = accumulator.to(Out.dtype.element_ty) + tl.store( + out_ptrs, + out, + mask=offs_d < D, + ) + + +class _JaggedDenseBroadcastAddFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + ): + jagged = switch_to_contiguous_if_needed(jagged) + dense = switch_to_contiguous_if_needed(dense) + L, D = jagged.shape + B, _ = dense.shape + out = torch.empty_like(jagged) + + grid = lambda meta: ( # noqa E731 + B, + triton.cdiv(max_seq_len, meta["BLOCK_N"]), + ) + BLOCK_D = triton.next_power_of_2(D) if D < 64 else 64 + jagged_dense_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=jagged, + Dense=dense, + Out=out, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + D=D, + stride_jn=jagged.stride(0), + stride_db=dense.stride(0), + stride_on=out.stride(0), + BLOCK_D=BLOCK_D, + ) + + ctx.save_for_backward(seq_offsets) + ctx.max_seq_len = max_seq_len + ctx.B = B + ctx.D = D + return out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, None, torch.Tensor, torch.Tensor]: + seq_offsets = ctx.saved_tensors[0] + d_dense = torch.empty((ctx.B, ctx.D), device=d_out.device, dtype=d_out.dtype) + BLOCK_D = triton.next_power_of_2(ctx.D) if ctx.D < 64 else 64 + jagged_reduce_sum[(ctx.B, triton.cdiv(ctx.D, BLOCK_D))]( + seq_offsets=seq_offsets, + Jagged=d_out, + Out=d_dense, + D=ctx.D, + stride_jn=d_out.stride(0), + stride_ob=d_dense.stride(0), + BLOCK_D=BLOCK_D, + ) + return None, None, d_out, d_dense + + +def triton_jagged_dense_bmm_add_fwd( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + elementwise: bool = False, +) -> Tuple[torch.Tensor, int, int, int]: + jagged = switch_to_contiguous_if_needed(jagged) + bias = switch_to_contiguous_if_needed(bias) + L, K = jagged.shape + B, _, N = dense.shape + out = torch.empty((L, N), dtype=jagged.dtype, device=jagged.device) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(N, meta["BLOCK_N"]), + triton.cdiv(max_seq_len, meta["BLOCK_M"]), + B, + ) + + jagged_dense_bmm_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=jagged, + Dense=dense, + Bias=bias, + Out=out, + AUTOTUNE_MAX_SEQ_LEN=fine_grained_autotune_max_seq_len(max_seq_len), + N=N, + K=K, + stride_jm=jagged.stride(0), + stride_db=dense.stride(0), + stride_dk=dense.stride(1), + stride_dn=dense.stride(2), + stride_bias_b=bias.stride(0), + stride_om=out.stride(0), + HAS_BIAS=True, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ELEMENTWISE=elementwise, + ) + + return out, B, K, N + + +def triton_jagged_dense_bmm_add_bwd_jagged( + max_seq_len: int, + seq_offsets: torch.Tensor, + d_jagged: torch.Tensor, + dense: torch.Tensor, + d_out: torch.Tensor, + K: int, + B: int, + N: int, +) -> torch.Tensor: + grid = lambda meta: ( # noqa E731 + triton.cdiv(K, meta["BLOCK_N"]), + triton.cdiv(max_seq_len, meta["BLOCK_M"]), + B, + ) + jagged_dense_bmm_broadcast_add_kernel[grid]( + seq_offsets=seq_offsets, + Jagged=d_out, + Dense=dense, + Bias=None, + Out=d_jagged, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + N=K, + K=N, + stride_jm=d_out.stride(0), + stride_db=dense.stride(0), + stride_dk=dense.stride(2), + stride_dn=dense.stride(1), + stride_bias_b=0, + stride_om=d_jagged.stride(0), + HAS_BIAS=False, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ELEMENTWISE=False, + ) + + return d_jagged + + +def triton_jagged_dense_bmm_add_bwd_dense_bias( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + d_dense: torch.Tensor, + B: int, + K: int, + N: int, + d_out: torch.Tensor, + elementwise: bool, +) -> Tuple[torch.Tensor, torch.Tensor]: + d_bias = torch.empty((B, N), device=d_out.device, dtype=d_out.dtype) + + grid = lambda meta: ( # noqa E731 + triton.cdiv(K, meta["BLOCK_M"]), + triton.cdiv(N, meta["BLOCK_N"]), + B, + ) + + if elementwise: + d_bias = d_out + reduce_out = None + stride_orb = 0 + stride_orn = 0 + reduce_jaggedb = False + else: + reduce_out = d_bias + stride_orb = d_bias.stride(0) + stride_orn = d_bias.stride(1) + reduce_jaggedb = True + + _jagged_jagged_bmm_reduce_sum[grid]( + seq_offsets=seq_offsets, + JaggedA=jagged, + JaggedB=d_out, + Out=d_dense, + ReduceOut=reduce_out, + M=K, + N=N, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + stride_ak=jagged.stride(0), + stride_bk=d_out.stride(0), + stride_ob=d_dense.stride(0), + stride_om=d_dense.stride(1), + stride_on=d_dense.stride(2), + stride_orb=stride_orb, + stride_orn=stride_orn, + REDUCE_JAGGEDB=reduce_jaggedb, + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ) + + return d_dense, d_bias + + +class _JaggedDenseBmmAddFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + elementwise: bool = False, + ): + if get_cuda_jagged_dense_bmm_fwd(): + jagged = switch_to_contiguous_if_needed(jagged) + bias = switch_to_contiguous_if_needed(bias) + # Ensure bias has same dtype as jagged (required by CUDA kernel) + bias = bias.to(jagged.dtype) + # Ensure seq_offsets is int64 (required by CUDA kernel) + seq_offsets = seq_offsets.to(torch.int64) + _, K = jagged.shape + B, _, N = dense.shape + out = torch.ops.jagged_dense_bmm_broadcast_add.jagged_dense_bmm_broadcast_add_fwd( + max_seq_len, seq_offsets, jagged, dense, bias, elementwise + ) + else: + out, B, K, N = triton_jagged_dense_bmm_add_fwd( + max_seq_len, seq_offsets, jagged, dense, bias, elementwise + ) + + ctx.save_for_backward(seq_offsets, jagged, dense) + ctx.B = B + ctx.max_seq_len = max_seq_len + ctx.K = K + ctx.N = N + ctx.elementwise = elementwise + return out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, None, torch.Tensor, torch.Tensor, torch.Tensor, None]: + seq_offsets, jagged, dense = ctx.saved_tensors + if get_cuda_jagged_dense_bmm_bwd(): + d_jagged, d_dense, d_bias = ( + torch.ops.jagged_dense_bmm_broadcast_add.jagged_dense_bmm_broadcast_add_bwd( + ctx.max_seq_len, + d_out, + seq_offsets.to(torch.int64), + jagged, + dense, + ctx.elementwise, + ) + ) + else: + d_jagged = triton_jagged_dense_bmm_add_bwd_jagged( + ctx.max_seq_len, + seq_offsets, + torch.empty_like(jagged), + dense, + d_out, + ctx.K, + ctx.B, + ctx.N, + ) + d_dense, d_bias = triton_jagged_dense_bmm_add_bwd_dense_bias( + ctx.max_seq_len, + seq_offsets, + jagged, + torch.empty_like(dense), + ctx.B, + ctx.K, + ctx.N, + d_out, + ctx.elementwise, + ) + + return None, None, d_jagged, d_dense, d_bias, None + + +@triton.jit +def concat_2D_jagged_w_prefix( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + n_prefix_from_B, # nonzero is not supported when IS_REPLACE=True + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + off_z = tl.program_id(1) + off_n = tl.program_id(0) + if IS_DENSE_A: + seq_start_a = off_z * DenseSize + seq_len_a = DenseSize + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + elif IS_DENSE_B: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = off_z * DenseSize + seq_len_b = DenseSize + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + + if IS_REPLACE: + seq_len = seq_len_a + else: + seq_len = seq_len_a + seq_len_b + if off_n >= seq_len: + return + + offs_d = tl.arange(0, BLOCK_D) + if IS_REPLACE: + out_seq_start = seq_start_a + off_n + out_seq_b_start = seq_len_a - seq_len_b + else: + out_seq_start = seq_start_a + seq_start_b + off_n + out_seq_b_start = seq_len_a + n_prefix_from_B + + out_ptrs = Out + out_seq_start.to(tl.int64) * stride_od + offs_d + if off_n < out_seq_b_start and off_n >= n_prefix_from_B: + off_a = off_n - n_prefix_from_B + if IS_DENSE_A: + in_ptrs = ( + ValuesA + + off_a.to(tl.int64) * stride_ad + + off_z.to(tl.int64) * stride_dense_batch + + offs_d + ) + else: + in_ptrs = ValuesA + (off_a + seq_start_a).to(tl.int64) * stride_ad + offs_d + else: + off_b = off_n - out_seq_b_start + n_prefix_from_B + if off_n < n_prefix_from_B: + off_b += out_seq_b_start - n_prefix_from_B + if IS_DENSE_B: + in_ptrs = ( + ValuesB + + off_b.to(tl.int64) * stride_bd + + off_z.to(tl.int64) * stride_dense_batch + + offs_d + ) + else: + in_ptrs = ValuesB + (off_b + seq_start_b).to(tl.int64) * stride_bd + offs_d + v = tl.load(in_ptrs, mask=offs_d < D) + tl.store(out_ptrs, v, mask=offs_d < D) + + +@triton.jit +def concat_2D_jagged( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + concat_2D_jagged_w_prefix( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + 0, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + IS_REPLACE, + ) + + +@triton.jit +def concat_2D_jagged_jagged_w_prefix( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + BLOCK_D: tl.constexpr, +): + concat_2D_jagged_w_prefix( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + 0, + Out, + D, + stride_ad, + stride_bd, + 0, + stride_od, + n_prefix_from_B, + IS_DENSE_A=False, + IS_DENSE_B=False, + BLOCK_D=BLOCK_D, + IS_REPLACE=False, + ) + + +@triton.jit +def split_2D_jagged_w_prefix( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + off_z = tl.program_id(1) + off_n = tl.program_id(0) + if IS_DENSE_A: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_start_a = off_z * DenseSize + seq_len_a = DenseSize + seq_len_b = seq_end_b - seq_start_b + elif IS_DENSE_B: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = off_z * DenseSize + seq_len_b = DenseSize + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + if IS_REPLACE: + seq_len = seq_len_a + else: + seq_len = seq_len_a + seq_len_b + if off_n >= seq_len: + return + + if IS_REPLACE: + seq_start = seq_start_a + out_seq_b_start = seq_len_a - seq_len_b + else: + seq_start = seq_start_a + seq_start_b + out_seq_b_start = seq_len_a + n_prefix_to_B + + offs_d = tl.arange(0, BLOCK_D) + in_ptrs = JaggedIn + (seq_start + off_n).to(tl.int64) * stride_id + offs_d + if off_n < out_seq_b_start and off_n >= n_prefix_to_B: + off_a = off_n - n_prefix_to_B + out_ptrs = OutA + (off_a + seq_start_a).to(tl.int64) * stride_ad + offs_d + else: + off_b = off_n - out_seq_b_start + n_prefix_to_B + if off_n < n_prefix_to_B: + off_b += out_seq_b_start - n_prefix_to_B + out_ptrs = OutB + (off_b + seq_start_b).to(tl.int64) * stride_bd + offs_d + v = tl.load(in_ptrs, mask=offs_d < D) + tl.store(out_ptrs, v, mask=offs_d < D) + + +@triton.jit +def split_2D_jagged( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + split_2D_jagged_w_prefix( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + 0, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + IS_REPLACE, + ) + + +@triton.jit +def split_2D_jagged_jagged_w_prefix( + JaggedIn, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + BLOCK_D: tl.constexpr, +): + split_2D_jagged_w_prefix( + JaggedIn, + 0, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A=False, + IS_DENSE_B=False, + BLOCK_D=BLOCK_D, + IS_REPLACE=False, + ) + + +def _triton_split_2D_jagged_internal( + jagged_in: torch.Tensor, + max_seq_len: int, + B: int, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + out_a: torch.Tensor, + out_b: torch.Tensor, + D: int, + dense_size: int, + n_prefix: int, + is_dense_a: bool, + is_dense_b: bool, + is_replace: bool, + BLOCK_D: int, +) -> None: + use_multirow = _should_use_multirow() + if n_prefix != 0: + if use_multirow: + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + split_2D_jagged_jagged_w_prefix_multirow[grid]( + JaggedIn=jagged_in, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + n_prefix_to_B=n_prefix, + BLOCK_D=BLOCK_D, + ) + else: + split_2D_jagged_jagged_w_prefix[(max_seq_len, B)]( + JaggedIn=jagged_in, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + n_prefix_to_B=n_prefix, + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + else: + if use_multirow: + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + split_2D_jagged_multirow[grid]( + JaggedIn=jagged_in, + DenseSize=dense_size, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + IS_REPLACE=is_replace, # pyre-ignore[6] + ) + else: + split_2D_jagged[(max_seq_len, B)]( + JaggedIn=jagged_in, + DenseSize=dense_size, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + IS_REPLACE=is_replace, # pyre-ignore[6] + ) + + +class _Concat2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + is_replace: bool = False, + n_prefix_from_right: int = 0, + ): + values_a = switch_to_contiguous_if_needed(values_a) + values_b = switch_to_contiguous_if_needed(values_b) + is_dense_a = offsets_a is None + is_dense_b = offsets_b is None + dense_size: int = 0 + if is_dense_a: + assert offsets_b is not None + B, dense_size, D = values_a.shape + seq_len_a = dense_size * B + seq_len_b, _ = values_b.shape + device = values_b.device + dtype = values_b.dtype + stride_dense_batch = values_a.stride(0) + elif is_dense_b: + assert offsets_a is not None + B, dense_size, D = values_b.shape + seq_len_a, _ = values_a.shape + seq_len_b = dense_size * B + device = values_a.device + dtype = values_a.dtype + stride_dense_batch = values_b.stride(0) + else: + assert offsets_a is not None and offsets_b is not None + B = offsets_a.shape[0] - 1 + seq_len_a, D = values_a.shape + seq_len_b, _ = values_b.shape + device = values_a.device + dtype = values_a.dtype + stride_dense_batch = 0 + + BLOCK_D = triton.next_power_of_2(D) + if is_replace: + values_out = torch.empty_like(values_a) + else: + values_out = torch.empty( + (seq_len_a + seq_len_b, D), device=device, dtype=dtype + ) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=values_out, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + D=D, + dense_size=dense_size, + stride_dense_batch=stride_dense_batch, + n_prefix=n_prefix_from_right, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + is_replace=is_replace, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.seq_len_a = seq_len_a + ctx.seq_len_b = seq_len_b + ctx.is_dense_a = is_dense_a + ctx.is_dense_b = is_dense_b + ctx.dense_size = dense_size + ctx.is_replace = is_replace + ctx.n_prefix_from_right = n_prefix_from_right + return values_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, torch.Tensor, torch.Tensor, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + is_dense_a, is_dense_b, is_replace = ( + ctx.is_dense_a, + ctx.is_dense_b, + ctx.is_replace, + ) + dense_size = ctx.dense_size + if is_dense_a: + B = offsets_b.shape[0] - 1 + else: + B = offsets_a.shape[0] - 1 + _, D = d_out.shape + BLOCK_D = triton.next_power_of_2(D) + values_a = torch.zeros( + (ctx.seq_len_a, D), device=d_out.device, dtype=d_out.dtype + ) + values_b = torch.empty( + (ctx.seq_len_b, D), device=d_out.device, dtype=d_out.dtype + ) + _triton_split_2D_jagged_internal( + jagged_in=d_out, + max_seq_len=ctx.max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + out_a=values_a, + out_b=values_b, + D=D, + dense_size=dense_size, + n_prefix=ctx.n_prefix_from_right, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + is_replace=is_replace, + BLOCK_D=BLOCK_D, + ) + + if is_dense_a: + values_a = values_a.reshape((B, dense_size, D)) + elif is_dense_b: + values_b = values_b.reshape((B, dense_size, D)) + return None, values_a, values_b, None, None, None, None + + +class _HelionConcat2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + ): + values_a = switch_to_contiguous_if_needed(values_a) + values_b = switch_to_contiguous_if_needed(values_b) + + assert offsets_a is not None and offsets_b is not None + B = offsets_a.shape[0] - 1 + seq_len_a, D = values_a.shape + seq_len_b, _ = values_b.shape + device = values_a.device + dtype = values_a.dtype + + BLOCK_D = triton.next_power_of_2(D) + values_out = torch.empty((seq_len_a + seq_len_b, D), device=device, dtype=dtype) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=values_out, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + D=D, + dense_size=0, + stride_dense_batch=0, + n_prefix=0, + is_dense_a=False, + is_dense_b=False, + is_replace=False, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.seq_len_a = seq_len_a + ctx.seq_len_b = seq_len_b + return values_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, torch.Tensor, torch.Tensor, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + d_out = switch_to_contiguous_if_needed(d_out) + values_a, values_b = _helion_split_2D_jagged_impl( + values=d_out, + max_seq_len=ctx.max_seq_len, + offsets_a=offsets_a, + offsets_b=offsets_b, + dense_size=0, + total_len_a=ctx.seq_len_a, + total_len_b=ctx.seq_len_b, + ) + + return None, values_a, values_b, None, None, None, None + + +class _Split2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + values: torch.Tensor, + max_seq_len: int, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + dense_size: int = 0, + n_prefix_to_right: int = 0, + seq_len_a: Optional[int] = None, + seq_len_b: Optional[int] = None, + total_seq_len: Optional[int] = None, + ) -> Tuple[torch.Tensor, torch.Tensor]: + values = switch_to_contiguous_if_needed(values) + is_dense_a: bool = offsets_a is None + is_dense_b: bool = offsets_b is None + if is_dense_a: + L, _ = values.shape + assert offsets_b is not None + B = offsets_b.shape[0] - 1 + seq_len_a = dense_size * B + seq_len_b = L - seq_len_a + offsets_a = offsets_b.new_empty(0) + elif is_dense_b: + L, _ = values.shape + assert offsets_a is not None + B = offsets_a.shape[0] - 1 + seq_len_b = dense_size * B + seq_len_a = L - seq_len_b + offsets_b = offsets_a.new_empty(0) + else: + assert offsets_a is not None and offsets_b is not None + B = offsets_a.shape[0] - 1 + + # Select the last offset item using torch.index_select instead of + # "int(offsets_a[-1].item())" so that it won't cause "Cannot cast + # FakeTensor to python number" error for AOTI. + if torch.compiler.is_compiling(): + offsets_b_last_idx = torch.tensor(offsets_b.size(0) - 1).to( + offsets_b.device, non_blocking=True + ) + if seq_len_b is None: + seq_len_b = offsets_b.index_select(dim=0, index=offsets_b_last_idx) + if seq_len_a is None and total_seq_len is None: + offsets_a_last_idx = torch.tensor(offsets_a.size(0) - 1).to( + offsets_a.device, non_blocking=True + ) + seq_len_a = offsets_a.index_select(dim=0, index=offsets_a_last_idx) + else: + if seq_len_b is None: + seq_len_b = int(offsets_b[-1].item()) + if seq_len_a is None and total_seq_len is None: + seq_len_a = int(offsets_a[-1].item()) + _, D = values.shape + BLOCK_D = triton.next_power_of_2(D) + # pyre-ignore[6] Incompatible parameter type + values_b = torch.empty((seq_len_b, D), device=values.device, dtype=values.dtype) + if seq_len_a is None: + # Derive seq_len_a from total_seq_len and values_b.size(0). + # values_b.size(0) is a SymInt (from the torch.empty above), + # so this is SymInt arithmetic — no new unbacked SymInt. + assert total_seq_len is not None + seq_len_a = total_seq_len - values_b.size(0) + # pyre-ignore[6] Incompatible parameter type + values_a = torch.empty((seq_len_a, D), device=values.device, dtype=values.dtype) + _triton_split_2D_jagged_internal( + jagged_in=values, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + out_a=values_a, + out_b=values_b, + D=D, + dense_size=dense_size, + n_prefix=n_prefix_to_right, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + is_replace=False, + BLOCK_D=BLOCK_D, + ) + if is_dense_a: + values_a = values_a.reshape(B, dense_size, D) + if is_dense_b: + values_b = values_b.reshape(B, dense_size, D) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.seq_len_a = seq_len_a + ctx.seq_len_b = seq_len_b + ctx.is_dense_a = is_dense_a + ctx.is_dense_b = is_dense_b + ctx.dense_size = dense_size + ctx.B = B + ctx.D = D + ctx.n_prefix_to_right = n_prefix_to_right + return values_a, values_b + + @staticmethod + def backward( + ctx, *d_values + ) -> Tuple[torch.Tensor, None, None, None, None, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + is_dense_a, is_dense_b = ctx.is_dense_a, ctx.is_dense_b + values_a, values_b = d_values + if is_dense_a: + stride_dense_batch = values_a.stride(0) + elif is_dense_b: + stride_dense_batch = values_b.stride(0) + else: + stride_dense_batch = 0 + + BLOCK_D = triton.next_power_of_2(ctx.D) + dvalues = torch.empty( + (ctx.seq_len_a + ctx.seq_len_b, ctx.D), + device=values_a.device, + dtype=values_b.dtype, + ) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=dvalues, + max_seq_len=ctx.max_seq_len, + B=ctx.B, + offsets_a=offsets_a, + offsets_b=offsets_b, + D=ctx.D, + dense_size=ctx.dense_size, + stride_dense_batch=stride_dense_batch, + n_prefix=ctx.n_prefix_to_right, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + is_replace=False, + BLOCK_D=BLOCK_D, + ) + + return dvalues, None, None, None, None, None, None, None, None + + +@torch.jit.unused +@torch.fx.wrap +def triton_jagged_dense_bmm_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, + bias: torch.Tensor, + elementwise: bool = False, +) -> torch.Tensor: + """ + Computing bmm Out = Jagged x Dense + Bias + M is the jagged dimension + Jagged has shape (sum_B(M_i), K), Dense has shape (B, K, N), Bias has shape (B, N) or (sum_B(M_i), N) depending on Elementwise, and Out has shape (sum_B(M_i), N) + """ + return _JaggedDenseBmmAddFunction.apply( + max_seq_len, seq_offsets, jagged, dense, bias, elementwise + ) + + +@torch.fx.wrap +def triton_concat_2D_jagged( + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + is_replace: bool = False, + n_prefix_from_right: int = 0, +) -> torch.Tensor: + return _Concat2DJaggedFunction.apply( + max_seq_len, + values_a, + values_b, + offsets_a, + offsets_b, + is_replace, + n_prefix_from_right, + ) + + +@torch.fx.wrap +def triton_concat_2D_jagged_jagged( + max_seq_len_left: int, + offsets_left: torch.Tensor, + values_left: torch.Tensor, + max_seq_len_right: int, + offsets_right: torch.Tensor, + values_right: torch.Tensor, + is_replace: bool, + n_prefix_from_right: int, +) -> torch.Tensor: + return triton_concat_2D_jagged( + max_seq_len=max_seq_len_left + max_seq_len_right, + values_a=values_left, + values_b=values_right, + offsets_a=offsets_left, + offsets_b=offsets_right, + is_replace=is_replace, + n_prefix_from_right=n_prefix_from_right, + ) + + +@torch.fx.wrap +def helion_concat_2D_jagged( + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, +) -> torch.Tensor: + return _HelionConcat2DJaggedFunction.apply( + max_seq_len, + values_a, + values_b, + offsets_a, + offsets_b, + ) + + +@torch.fx.wrap +def triton_concat_2D_dense_jagged( + jagged_max_seq_len: int, + jagged_offsets: torch.Tensor, + jagged_values: torch.Tensor, + dense_values: torch.Tensor, +) -> torch.Tensor: + B, dense_size, D = dense_values.size() + max_seq_len = jagged_max_seq_len + dense_size + return triton_concat_2D_jagged( + max_seq_len=max_seq_len, + values_a=dense_values, + values_b=jagged_values, + offsets_a=None, + offsets_b=jagged_offsets, + ) + + +def triton_jagged_dense_bmm( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, +) -> torch.Tensor: + return _JaggedDenseBmmFunction.apply(max_seq_len, seq_offsets, jagged, dense) + + +@torch.jit.unused +def triton_split_2D_jagged( + values: torch.Tensor, + max_seq_len: int, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + dense_size: int = 0, + n_prefix_to_right: int = 0, + seq_len_a: Optional[int] = None, + seq_len_b: Optional[int] = None, + total_seq_len: Optional[int] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _Split2DJaggedFunction.apply( + values, + max_seq_len, + offsets_a, + offsets_b, + dense_size, + n_prefix_to_right, + seq_len_a, + seq_len_b, + total_seq_len, + ) + + +@torch.jit.unused +def helion_split_2D_jagged( + values: torch.Tensor, + max_seq_len: int, + offsets_a: Optional[torch.Tensor] = None, + offsets_b: Optional[torch.Tensor] = None, + dense_size: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _HelionSplit2DJaggedFunction.apply( + values, + max_seq_len, + offsets_a, + offsets_b, + dense_size, + ) + + +@triton.jit +def concat_2D_jagged_w_prefix_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + n_prefix_from_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + off_z = tl.program_id(1) + off_block_n = tl.program_id(0) + + if IS_DENSE_A: + seq_start_a = off_z * DenseSize + seq_len_a = DenseSize + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + elif IS_DENSE_B: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = off_z * DenseSize + seq_len_b = DenseSize + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + + if IS_REPLACE: + seq_len = seq_len_a + out_seq_start = seq_start_a + out_seq_b_start = seq_len_a - seq_len_b + else: + seq_len = seq_len_a + seq_len_b + out_seq_start = seq_start_a + seq_start_b + out_seq_b_start = seq_len_a + n_prefix_from_B + + start_n = off_block_n * BLOCK_N + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + if start_n >= seq_len: + return + valid_mask = offs_n < seq_len + + out_ptrs = ( + Out + + (out_seq_start + offs_n[:, None]).to(tl.int64) * stride_od + + offs_d[None, :] + ) + + to_a_mask = (offs_n < out_seq_b_start) & (offs_n >= n_prefix_from_B) & valid_mask + to_b_mask = ~to_a_mask & valid_mask + + off_a = offs_n - n_prefix_from_B + if IS_DENSE_A: + in_a_ptrs = ( + ValuesA + + off_a[:, None].to(tl.int64) * stride_ad + + off_z.to(tl.int64) * stride_dense_batch + + offs_d[None, :] + ) + else: + in_a_ptrs = ( + ValuesA + + (off_a[:, None] + seq_start_a).to(tl.int64) * stride_ad + + offs_d[None, :] + ) + + v_a = tl.load(in_a_ptrs, mask=to_a_mask[:, None] & (offs_d[None, :] < D), other=0.0) + tl.store(out_ptrs, v_a, mask=to_a_mask[:, None] & (offs_d[None, :] < D)) + + prefix_mask = offs_n < n_prefix_from_B + + off_b = tl.where(prefix_mask, offs_n, offs_n - out_seq_b_start + n_prefix_from_B) + if IS_DENSE_B: + in_b_ptrs = ( + ValuesB + + off_b[:, None].to(tl.int64) * stride_bd + + off_z.to(tl.int64) * stride_dense_batch + + offs_d[None, :] + ) + else: + in_b_ptrs = ( + ValuesB + + (off_b[:, None] + seq_start_b).to(tl.int64) * stride_bd + + offs_d[None, :] + ) + + v_b = tl.load(in_b_ptrs, mask=to_b_mask[:, None] & (offs_d[None, :] < D), other=0.0) + tl.store(out_ptrs, v_b, mask=to_b_mask[:, None] & (offs_d[None, :] < D)) + + +@triton_autotune( + configs=_get_split_concat_2d_jagged_multirow_configs_wrapper(), + key=["BLOCK_D"], +) +@triton.jit +def concat_2D_jagged_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + concat_2D_jagged_w_prefix_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + DenseSize, + Out, + D, + stride_ad, + stride_bd, + stride_dense_batch, + stride_od, + 0, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + BLOCK_N, + IS_REPLACE, + ) + + +@triton_autotune( + configs=_get_split_concat_2d_jagged_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def concat_2D_jagged_jagged_w_prefix_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + concat_2D_jagged_w_prefix_multirow( + OffsetsA, + ValuesA, + OffsetsB, + ValuesB, + 0, + Out, + D, + stride_ad, + stride_bd, + 0, + stride_od, + n_prefix_from_B, + IS_DENSE_A=False, + IS_DENSE_B=False, + BLOCK_D=BLOCK_D, + BLOCK_N=BLOCK_N, + IS_REPLACE=False, + ) + + +@triton.jit +def split_2D_jagged_w_prefix_multirow( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + off_z = tl.program_id(1) + off_block_n = tl.program_id(0) + + if IS_DENSE_A: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_start_a = off_z * DenseSize + seq_len_a = DenseSize + seq_len_b = seq_end_b - seq_start_b + elif IS_DENSE_B: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = off_z * DenseSize + seq_len_b = DenseSize + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + + if IS_REPLACE: + seq_len = seq_len_a + else: + seq_len = seq_len_a + seq_len_b + + if IS_REPLACE: + seq_start = seq_start_a + out_seq_b_start = seq_len_a - seq_len_b + else: + seq_start = seq_start_a + seq_start_b + out_seq_b_start = seq_len_a + n_prefix_to_B + + start_n = off_block_n * BLOCK_N + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + if start_n >= seq_len: + return + valid_mask = offs_n < seq_len + + in_ptrs = ( + JaggedIn + + (seq_start + offs_n[:, None]).to(tl.int64) * stride_id + + offs_d[None, :] + ) + + v = tl.load(in_ptrs, mask=valid_mask[:, None] & (offs_d[None, :] < D), other=0.0) + + to_a_mask = (offs_n < out_seq_b_start) & (offs_n >= n_prefix_to_B) & valid_mask + to_b_mask = ~to_a_mask & valid_mask + + off_a = offs_n - n_prefix_to_B + out_a_ptrs = ( + OutA + (off_a[:, None] + seq_start_a).to(tl.int64) * stride_ad + offs_d[None, :] + ) + tl.store(out_a_ptrs, v, mask=to_a_mask[:, None] & (offs_d[None, :] < D)) + + prefix_mask = offs_n < n_prefix_to_B + + off_b = tl.where(prefix_mask, offs_n, offs_n - out_seq_b_start + n_prefix_to_B) + out_b_ptrs = ( + OutB + (off_b[:, None] + seq_start_b).to(tl.int64) * stride_bd + offs_d[None, :] + ) + tl.store(out_b_ptrs, v, mask=to_b_mask[:, None] & (offs_d[None, :] < D)) + + +@triton_autotune( + configs=pinned_or_full( + [triton.Config({"BLOCK_N": 8}, num_warps=1)], + _get_split_concat_2d_jagged_multirow_configs_wrapper, + ), + key=["BLOCK_D"], +) +@triton.jit +def split_2D_jagged_multirow( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + IS_REPLACE: tl.constexpr, +): + split_2D_jagged_w_prefix_multirow( + JaggedIn, + DenseSize, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + 0, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + BLOCK_N, + IS_REPLACE, + ) + + +@triton_autotune( + configs=_get_split_concat_2d_jagged_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def split_2D_jagged_jagged_w_prefix_multirow( + JaggedIn, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + split_2D_jagged_w_prefix_multirow( + JaggedIn, + 0, + OffsetsA, + OffsetsB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A=False, + IS_DENSE_B=False, + BLOCK_D=BLOCK_D, + BLOCK_N=BLOCK_N, + IS_REPLACE=False, + ) + + +def triton_jagged_dense_broadcast_add( + max_seq_len: int, + seq_offsets: torch.Tensor, + jagged: torch.Tensor, + dense: torch.Tensor, +) -> torch.Tensor: + return _JaggedDenseBroadcastAddFunction.apply( + max_seq_len, seq_offsets, jagged, dense + ) + + +@triton.jit +def _helion_split_2d_jagged_kernel( + offsets_a, + offsets_b, + values_flat, + out_a_flat, + out_b_flat, + max_seq_len, + D: tl.constexpr, + _BLOCK_SIZE_0: tl.constexpr, + _BLOCK_SIZE_1: tl.constexpr, +) -> None: + # Get program ID and decompose to batch and sequence block coordinates + program_id = tl.program_id(0) + flat_program_id = program_id + batch_id = triton_helpers.div_floor_integer( + flat_program_id, + triton_helpers.div_floor_integer( + -1 + _BLOCK_SIZE_0 + max_seq_len, _BLOCK_SIZE_0 + ), + ) + seq_block_id = triton_helpers.remainder_integer( # noqa: F841 + flat_program_id, + triton_helpers.div_floor_integer( + -1 + _BLOCK_SIZE_0 + max_seq_len, _BLOCK_SIZE_0 + ), + ) + # Load output boundaries for part A + out_a_start = tl.load(offsets_a + batch_id * 1, None, eviction_policy="evict_last") + batch_id_plus_1 = 1 + triton_helpers.div_floor_integer( + flat_program_id, + triton_helpers.div_floor_integer( + -1 + _BLOCK_SIZE_0 + max_seq_len, _BLOCK_SIZE_0 + ), + ) + out_a_end = tl.load( + offsets_a + batch_id_plus_1 * 1, None, eviction_policy="evict_last" + ) + len_a = out_a_end - out_a_start + # Load output boundaries for part B + out_b_start = tl.load(offsets_b + batch_id * 1, None) + out_b_end = tl.load( + offsets_b + batch_id_plus_1 * 1, None, eviction_policy="evict_last" + ) + len_b = out_b_end - out_b_start + # Compute input start and total length for this batch + input_start = out_a_start + out_b_start + total_len = len_a + len_b + # Calculate sequence offset for this block + seq_offset = _BLOCK_SIZE_0 * triton_helpers.remainder_integer( + flat_program_id, + triton_helpers.div_floor_integer( + -1 + _BLOCK_SIZE_0 + max_seq_len, _BLOCK_SIZE_0 + ), + ) + has_work = total_len > seq_offset + if has_work: + # Generate row indices for this sequence block + seq_range = tl.arange(0, _BLOCK_SIZE_0) + seq_offset_i32 = tl.cast(seq_offset, tl.int32) + row_indices = seq_range + seq_offset_i32 + + # Create masks for valid rows and parts A/B + total_len_i32 = tl.cast(total_len[None], tl.int32) + len_a_i32 = tl.cast(len_a[None], tl.int32) + valid_mask = row_indices < total_len_i32 + is_part_a = row_indices < len_a_i32 + is_part_b = (row_indices >= len_a_i32) & valid_mask + + # Extract scalar values once + input_start_i32 = tl.cast(input_start[None, None], tl.int32) + out_a_start_i32 = tl.cast(out_a_start[None, None], tl.int32) + out_b_start_i32 = tl.cast(out_b_start[None, None], tl.int32) + + # Process features in smaller tiles + for feature_offset in tl.range( + 0, + D, + _BLOCK_SIZE_1, + loop_unroll_factor=1, + num_stages=4, + disallow_acc_multi_buffer=True, + flatten=True, + ): + feature_indices = feature_offset + tl.arange(0, _BLOCK_SIZE_1).to(tl.int32) + + # Compute D constant and feature mask once per feature iteration + D_const = tl.full([], tl.cast(D, tl.int32), tl.int32) + D_i32 = tl.cast(D, tl.int32) + feature_mask = feature_indices < D_i32 + + # Compute indices for part A + row_subscript = row_indices[:, None] + input_row_a = input_start_i32 + row_subscript + input_idx_a = ( + tl.cast(input_row_a * D_const, tl.int32) + feature_indices[None, :] + ) + + out_a_row = out_a_start_i32 + row_subscript + out_a_idx = ( + tl.cast(out_a_row * D_const, tl.int32) + feature_indices[None, :] + ) + + mask_a = is_part_a[:, None] & valid_mask[:, None] & feature_mask[None, :] + + # Load and store part A data + slice_a = tl.load( + values_flat + input_idx_a * 1, + mask_a, + other=0, + eviction_policy="evict_first", + ) + tl.store(out_a_flat + out_a_idx * 1, slice_a, mask_a) + + # Compute indices for part B + input_idx_b = ( + tl.cast((input_start_i32 + row_subscript) * D_const, tl.int32) + + feature_indices[None, :] + ) + + row_minus_len_a = row_subscript - len_a_i32 + out_b_row = out_b_start_i32 + row_minus_len_a + out_b_idx = ( + tl.cast(out_b_row * D_const, tl.int32) + feature_indices[None, :] + ) + + mask_b = is_part_b[:, None] & feature_mask[None, :] + + # Load and store part B data + slice_b = tl.load( + values_flat + input_idx_b * 1, + mask_b, + other=0, + eviction_policy="evict_first", + ) + tl.store(out_b_flat + out_b_idx * 1, slice_b, mask_b) + + +class _HelionSplit2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + values: torch.Tensor, + max_seq_len: int, + offsets_a: torch.Tensor, + offsets_b: torch.Tensor, + dense_size: int = 0, # noqa: F841 + ) -> Tuple[torch.Tensor, torch.Tensor]: + values = switch_to_contiguous_if_needed(values) + B = offsets_a.shape[0] - 1 + D = values.size(1) + + # TODO: maybe check if torch.compiler.is_compiling() and use index_select instead + seq_len_a = int(offsets_a[-1].item()) + seq_len_b = int(offsets_b[-1].item()) + + values_a, values_b = _helion_split_2D_jagged_impl( + values=values, + max_seq_len=max_seq_len, + offsets_a=offsets_a, + offsets_b=offsets_b, + dense_size=dense_size, + total_len_a=seq_len_a, + total_len_b=seq_len_b, + ) + + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.seq_len_a = seq_len_a + ctx.seq_len_b = seq_len_b + ctx.dense_size = dense_size + ctx.B = B + ctx.D = D + return values_a, values_b + + @staticmethod + def backward(ctx, *d_values) -> Tuple[torch.Tensor, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + values_a, values_b = d_values + BLOCK_D = triton.next_power_of_2(ctx.D) + + dvalues = torch.empty( + (ctx.seq_len_a + ctx.seq_len_b, ctx.D), + device=values_a.device, + dtype=values_a.dtype, + ) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=dvalues, + max_seq_len=ctx.max_seq_len, + B=ctx.B, + offsets_a=offsets_a, + offsets_b=offsets_b, + D=ctx.D, + dense_size=0, + stride_dense_batch=0, + n_prefix=0, + is_dense_a=False, + is_dense_b=False, + is_replace=False, + BLOCK_D=BLOCK_D, + ) + return dvalues, None, None, None, None + + +def _helion_split_2D_jagged_impl( + values: torch.Tensor, + max_seq_len: int, + offsets_a: torch.Tensor, + offsets_b: torch.Tensor, + dense_size: int = 0, # noqa: F841 + total_len_a: Optional[int] = None, + total_len_b: Optional[int] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + D = values.size(1) + + # Select dtype-specific optimal parameters + if values.dtype == torch.float32: + # FP32-optimized parameters + block_size_0 = 64 + block_size_1 = 64 + num_warps = 4 + num_stages = 4 + else: + # BF16/FP16-optimized parameters + block_size_0 = 128 + block_size_1 = triton.next_power_of_2(D) + num_warps = 32 + num_stages = 7 + + return _helion_split_2d_jagged( + values, + max_seq_len, + offsets_a, + offsets_b, + dense_size, + block_size_0=block_size_0, + block_size_1=block_size_1, + num_warps=num_warps, + num_stages=num_stages, + total_len_a=total_len_a, + total_len_b=total_len_b, + ) + + +def _helion_split_2d_jagged( + values: torch.Tensor, + max_seq_len: int, + offsets_a: torch.Tensor, + offsets_b: torch.Tensor, + dense_size: int, # noqa: F841 + block_size_0: int = 64, + block_size_1: int = 64, + num_warps: int = 4, + num_stages: int = 4, + total_len_a: Optional[int] = None, + total_len_b: Optional[int] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + values = values.contiguous() + num_batches = offsets_a.size(0) - 1 + D = values.size(1) + num_seq_blocks = (max_seq_len + block_size_0 - 1) // block_size_0 + if total_len_a is None: + total_len_a = int(offsets_a[-1].item()) + if total_len_b is None: + total_len_b = int(offsets_b[-1].item()) + out_a = torch.empty([total_len_a, D], dtype=values.dtype, device=values.device) + out_b = torch.empty([total_len_b, D], dtype=values.dtype, device=values.device) + values_flat = values.view(-1) + out_a_flat = out_a.view(-1) + out_b_flat = out_b.view(-1) + total_programs = num_batches * num_seq_blocks + + # pyre-ignore[28] + _helion_split_2d_jagged_kernel[(total_programs,)]( + offsets_a, + offsets_b, + values_flat, + out_a_flat, + out_b_flat, + max_seq_len, + D, + block_size_0, + block_size_1, + num_warps=num_warps, + num_stages=num_stages, + ) + return (out_a, out_b) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py new file mode 100644 index 000000000..2c3728c0a --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_jagged_tensors.py @@ -0,0 +1,1088 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +from typing import Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.utils import is_sm100_plus + + +def _triton_concat_2D_jagged_internal( + values_a: torch.Tensor, + values_b: torch.Tensor, + values_out: torch.Tensor, + max_seq_len: int, + B: int, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + max_len_a: Optional[int], + max_len_b: Optional[int], + D: int, + n_prefix_from_B: int, + is_dense_a: bool, + is_dense_b: bool, + BLOCK_D: int, +) -> None: + if is_sm100_plus() or (torch.cuda.is_available() and torch.version.hip): + # Route AMD/ROCm through the multirow kernel. + # + # The basic `_concat_2D_jagged` kernel below issues one program per + # output row (grid = `(max_seq_len, B)`). On ROCm Triton this fails to + # lower in the `TritonAMDGPUCanonicalizePointers` pass with + # `RuntimeError: PassManager::run failed` at `make_ttgir`. The + # multirow variant tiles rows with a tunable `BLOCK_N` (grid = + # `(cdiv(max_seq_len, BLOCK_N), B)`) and compiles cleanly on ROCm. + # The original `is_sm100_plus()` gate was conservative — only Blackwell + # was opted in. Adding HIP keeps NVIDIA H100/A100 on the basic kernel + # they were validated against and unblocks AMD without behavior change + # on existing NVIDIA paths. + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + concat_2D_jagged_multirow[grid]( + ValuesA=values_a, + ValuesB=values_b, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=max_len_a, + MaxLenB=max_len_b, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_od=values_out.stride(-2), + n_prefix_from_B=n_prefix_from_B, + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + else: + _concat_2D_jagged[(max_seq_len, B)]( + ValuesA=values_a, + ValuesB=values_b, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=max_len_a, + MaxLenB=max_len_b, + Out=values_out, + D=D, + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + stride_od=values_out.stride(-2), + n_prefix_from_B=n_prefix_from_B, + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + + +def _triton_split_2D_jagged_internal( + jagged_in: torch.Tensor, + max_seq_len: int, + B: int, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + max_len_a: Optional[int], + max_len_b: Optional[int], + out_a: torch.Tensor, + out_b: torch.Tensor, + D: int, + n_prefix_to_B: int, + is_dense_a: bool, + is_dense_b: bool, + BLOCK_D: int, +) -> None: + if is_sm100_plus() or (torch.cuda.is_available() and torch.version.hip): + # Route AMD/ROCm through the multirow kernel for the same reason as + # `_triton_concat_2D_jagged_internal` above: basic `_split_2D_jagged` + # hits `PassManager::run failed` in `TritonAMDGPUCanonicalizePointers` + # on ROCm; multirow lowers cleanly. + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + split_2D_jagged_multirow[grid]( + JaggedIn=jagged_in, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=max_len_a, + MaxLenB=max_len_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + n_prefix_to_B=n_prefix_to_B, + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + else: + _split_2D_jagged[(max_seq_len, B)]( + JaggedIn=jagged_in, + OffsetsA=offsets_a, + OffsetsB=offsets_b, + MaxLenA=max_len_a, + MaxLenB=max_len_b, + OutA=out_a, + OutB=out_b, + D=D, + stride_id=jagged_in.stride(0), + stride_ad=out_a.stride(0), + stride_bd=out_b.stride(0), + n_prefix_to_B=n_prefix_to_B, + IS_DENSE_A=is_dense_a, # pyre-ignore[6] + IS_DENSE_B=is_dense_b, # pyre-ignore[6] + BLOCK_D=BLOCK_D, # pyre-ignore[6] + ) + + +def _get_concat_split_2d_jagged_multirow_configs(): + configs = [] + for BLOCK_N in [1, 2, 4, 8]: + for num_warps in [1, 2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton.jit +def _concat_2D_jagged_multirow( + ValuesA, + ValuesB, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + off_z = tl.program_id(1) + block_n = tl.program_id(0) + + if IS_DENSE_A: + seq_start_a = off_z * MaxLenA + seq_len_a = MaxLenA + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + if IS_DENSE_B: + seq_start_b = off_z * MaxLenB + seq_len_b = MaxLenB + else: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + seq_len = seq_len_a + seq_len_b + + start_n = block_n * BLOCK_N + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + + valid_mask = offs_n < seq_len + + out_seq_start = seq_start_a + seq_start_b + offs_n + out_ptrs = Out + out_seq_start[:, None].to(tl.int64) * stride_od + offs_d[None, :] + + from_prefix_b_mask = (offs_n < n_prefix_from_B) & valid_mask + from_a_mask = ( + (offs_n >= n_prefix_from_B) + & (offs_n < seq_len_a + n_prefix_from_B) + & valid_mask + ) + from_suffix_b_mask = (offs_n >= seq_len_a + n_prefix_from_B) & valid_mask + + in_b1_ptrs = ( + ValuesB + + (offs_n[:, None] + seq_start_b).to(tl.int64) * stride_bd + + offs_d[None, :] + ) + v_b1 = tl.load( + in_b1_ptrs, mask=from_prefix_b_mask[:, None] & (offs_d[None, :] < D), other=0.0 + ) + tl.store(out_ptrs, v_b1, mask=from_prefix_b_mask[:, None] & (offs_d[None, :] < D)) + + off_a = offs_n - n_prefix_from_B + in_a_ptrs = ( + ValuesA + + (off_a[:, None] + seq_start_a).to(tl.int64) * stride_ad + + offs_d[None, :] + ) + v_a = tl.load( + in_a_ptrs, mask=from_a_mask[:, None] & (offs_d[None, :] < D), other=0.0 + ) + tl.store(out_ptrs, v_a, mask=from_a_mask[:, None] & (offs_d[None, :] < D)) + + off_b = offs_n - seq_len_a + in_b2_ptrs = ( + ValuesB + + (off_b[:, None] + seq_start_b).to(tl.int64) * stride_bd + + offs_d[None, :] + ) + v_b2 = tl.load( + in_b2_ptrs, mask=from_suffix_b_mask[:, None] & (offs_d[None, :] < D), other=0.0 + ) + tl.store(out_ptrs, v_b2, mask=from_suffix_b_mask[:, None] & (offs_d[None, :] < D)) + + +@triton_autotune( + configs=_get_concat_split_2d_jagged_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def concat_2D_jagged_multirow( + ValuesA, + ValuesB, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + _concat_2D_jagged_multirow( + ValuesA, + ValuesB, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + BLOCK_N, + ) + + +@triton.jit +def _split_2D_jagged_multirow( + JaggedIn, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + off_z = tl.program_id(1) + block_n = tl.program_id(0) + + if IS_DENSE_A: + seq_start_a = off_z * MaxLenA + seq_len_a = MaxLenA + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + if IS_DENSE_B: + seq_start_b = off_z * MaxLenB + seq_len_b = MaxLenB + else: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + seq_len = seq_len_a + seq_len_b + seq_start = seq_start_a + seq_start_b + + start_n = block_n * BLOCK_N + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + + valid_mask = offs_n < seq_len + + in_ptrs = ( + JaggedIn + + (seq_start + offs_n[:, None]).to(tl.int64) * stride_id + + offs_d[None, :] + ) + + v = tl.load(in_ptrs, mask=valid_mask[:, None] & (offs_d[None, :] < D), other=0.0) + + to_prefix_b_mask = (offs_n < n_prefix_to_B) & valid_mask + to_a_mask = ( + (offs_n >= n_prefix_to_B) & (offs_n < seq_len_a + n_prefix_to_B) & valid_mask + ) + to_suffix_b_mask = (offs_n >= seq_len_a + n_prefix_to_B) & valid_mask + + out_b1_ptrs = ( + OutB + + (offs_n[:, None] + seq_start_b).to(tl.int64) * stride_bd + + offs_d[None, :] + ) + tl.store(out_b1_ptrs, v, mask=to_prefix_b_mask[:, None] & (offs_d[None, :] < D)) + + off_a = offs_n - n_prefix_to_B + out_a_ptrs = ( + OutA + (off_a[:, None] + seq_start_a).to(tl.int64) * stride_ad + offs_d[None, :] + ) + tl.store(out_a_ptrs, v, mask=to_a_mask[:, None] & (offs_d[None, :] < D)) + + off_b = offs_n - seq_len_a + out_b2_ptrs = ( + OutB + (off_b[:, None] + seq_start_b).to(tl.int64) * stride_bd + offs_d[None, :] + ) + tl.store(out_b2_ptrs, v, mask=to_suffix_b_mask[:, None] & (offs_d[None, :] < D)) + + +@triton_autotune( + configs=_get_concat_split_2d_jagged_multirow_configs(), + key=["BLOCK_D"], +) +@triton.jit +def split_2D_jagged_multirow( + JaggedIn, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + _split_2D_jagged_multirow( + JaggedIn, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A, + IS_DENSE_B, + BLOCK_D, + BLOCK_N, + ) + + +@triton.jit +def _concat_2D_jagged( + ValuesA, + ValuesB, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + Out, + D, + stride_ad, + stride_bd, + stride_od, + n_prefix_from_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, +): + off_z = tl.program_id(1) + off_n = tl.program_id(0) + if IS_DENSE_A: + seq_start_a = off_z * MaxLenA + seq_len_a = MaxLenA + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + if IS_DENSE_B: + seq_start_b = off_z * MaxLenB + seq_len_b = MaxLenB + else: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + seq_len = seq_len_a + seq_len_b + if off_n >= seq_len: + return + offs_d = tl.arange(0, BLOCK_D) + out_seq_start = seq_start_a + seq_start_b + off_n + out_ptrs = Out + out_seq_start.to(tl.int64) * stride_od + offs_d + if off_n < n_prefix_from_B: + in_ptrs = ValuesB + (off_n + seq_start_b).to(tl.int64) * stride_bd + offs_d + elif off_n < seq_len_a + n_prefix_from_B: + in_ptrs = ( + ValuesA + + (off_n - n_prefix_from_B + seq_start_a).to(tl.int64) * stride_ad + + offs_d + ) + else: + in_ptrs = ( + ValuesB + + (off_n - seq_len_a + seq_start_b).to(tl.int64) * stride_bd + + offs_d + ) + v = tl.load(in_ptrs, mask=offs_d < D) + tl.store(out_ptrs, v, mask=offs_d < D) + + +@triton.jit +def _split_2D_jagged( + JaggedIn, + OffsetsA, + OffsetsB, + MaxLenA, + MaxLenB, + OutA, + OutB, + D, + stride_id, + stride_ad, + stride_bd, + n_prefix_to_B, + IS_DENSE_A: tl.constexpr, + IS_DENSE_B: tl.constexpr, + BLOCK_D: tl.constexpr, +): + off_z = tl.program_id(1) + off_n = tl.program_id(0) + if IS_DENSE_A: + seq_start_a = off_z * MaxLenA + seq_len_a = MaxLenA + else: + seq_start_a = tl.load(OffsetsA + off_z) + seq_end_a = tl.load(OffsetsA + off_z + 1) + seq_len_a = seq_end_a - seq_start_a + if IS_DENSE_B: + seq_start_b = off_z * MaxLenB + seq_len_b = MaxLenB + else: + seq_start_b = tl.load(OffsetsB + off_z) + seq_end_b = tl.load(OffsetsB + off_z + 1) + seq_len_b = seq_end_b - seq_start_b + seq_len = seq_len_a + seq_len_b + if off_n >= seq_len: + return + seq_start = seq_start_a + seq_start_b + offs_d = tl.arange(0, BLOCK_D) + in_ptrs = JaggedIn + (seq_start + off_n).to(tl.int64) * stride_id + offs_d + if off_n < n_prefix_to_B: + out_ptrs = OutB + (off_n + seq_start_b).to(tl.int64) * stride_bd + offs_d + elif off_n < seq_len_a + n_prefix_to_B: + out_ptrs = ( + OutA + + (off_n - n_prefix_to_B + seq_start_a).to(tl.int64) * stride_ad + + offs_d + ) + else: + out_ptrs = ( + OutB + (off_n - seq_len_a + seq_start_b).to(tl.int64) * stride_bd + offs_d + ) + v = tl.load(in_ptrs, mask=offs_d < D) + tl.store(out_ptrs, v, mask=offs_d < D) + + +class _Concat2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + max_len_a: Optional[int], + max_len_b: Optional[int], + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + n_prefix_from_B: int, + ): + values_a = switch_to_contiguous_if_needed(values_a) + values_b = switch_to_contiguous_if_needed(values_b) + is_dense_a = offsets_a is None + is_dense_b = offsets_b is None + total_len_a, D = values_a.shape + total_len_b, _ = values_b.shape + if is_dense_a: + assert max_len_a is not None + B = total_len_a // max_len_a + else: + assert offsets_a is not None + B = offsets_a.shape[0] - 1 + if is_dense_b: + assert max_len_b is not None + B = total_len_b // max_len_b + else: + assert offsets_b is not None + B = offsets_b.shape[0] - 1 + total_seq_len = total_len_a + total_len_b + BLOCK_D = triton.next_power_of_2(D) + values_out = torch.empty( + (total_seq_len, D), device=values_a.device, dtype=values_a.dtype + ) + _triton_concat_2D_jagged_internal( + values_a=values_a, + values_b=values_b, + values_out=values_out, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + max_len_a=max_len_a, + max_len_b=max_len_b, + D=D, + n_prefix_from_B=n_prefix_from_B, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.total_len_a = total_len_a + ctx.total_len_b = total_len_b + ctx.is_dense_a = is_dense_a + ctx.is_dense_b = is_dense_b + ctx.max_len_a = max_len_a + ctx.max_len_b = max_len_b + ctx.B = B + ctx.n_prefix_from_B = n_prefix_from_B + return values_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, torch.Tensor, torch.Tensor, None, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + _, D = d_out.shape + BLOCK_D = triton.next_power_of_2(D) + d_values_a = torch.zeros( + (ctx.total_len_a, D), device=d_out.device, dtype=d_out.dtype + ) + d_values_b = torch.empty( + (ctx.total_len_b, D), device=d_out.device, dtype=d_out.dtype + ) + # Go through `_triton_split_2D_jagged_internal` (not raw + # `_split_2D_jagged[grid]`) so this backward pass benefits from the same + # AMD-routing-through-multirow workaround as the forward. Calling the + # raw kernel directly would hit `PassManager::run failed` on ROCm at + # `TritonAMDGPUCanonicalizePointers`. If you refactor this, do not + # collapse it back to `_split_2D_jagged[(ctx.max_seq_len, ctx.B)](...)`. + _triton_split_2D_jagged_internal( + jagged_in=d_out, + max_seq_len=ctx.max_seq_len, + B=ctx.B, + offsets_a=offsets_a, + offsets_b=offsets_b, + max_len_a=ctx.max_len_a, + max_len_b=ctx.max_len_b, + out_a=d_values_a, + out_b=d_values_b, + D=D, + n_prefix_to_B=ctx.n_prefix_from_B, + is_dense_a=ctx.is_dense_a, + is_dense_b=ctx.is_dense_b, + BLOCK_D=BLOCK_D, + ) + return None, d_values_a, d_values_b, None, None, None, None, None + + +class _Split2DJaggedFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int], + total_len_right: Optional[int], + max_len_a: Optional[int], + max_len_b: Optional[int], + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + n_prefix_to_B: int, + ) -> Tuple[torch.Tensor, torch.Tensor]: + values = switch_to_contiguous_if_needed(values) + is_dense_a: bool = offsets_a is None + is_dense_b: bool = offsets_b is None + total_seq_len, D = values.shape + if is_dense_a: + assert is_dense_b is False + assert offsets_b is not None + assert max_len_a is not None + B = offsets_b.shape[0] - 1 + total_len_a = max_len_a * B + total_len_b = total_seq_len - total_len_a + elif is_dense_b: + assert is_dense_a is False + assert offsets_a is not None + assert max_len_b is not None + B = offsets_a.shape[0] - 1 + total_len_b = max_len_b * B + total_len_a = total_seq_len - total_len_b + else: + assert offsets_a is not None and offsets_b is not None + B = offsets_a.shape[0] - 1 + if total_len_left is not None and total_len_right is not None: + assert total_len_left + total_len_right == total_seq_len + total_len_a = total_len_left + total_len_b = total_len_right + else: + total_len_a = int(offsets_a[-1].item()) + total_len_b = values.size(0) - total_len_a + _, D = values.shape + BLOCK_D = triton.next_power_of_2(D) + values_a = torch.empty( + (total_len_a, D), device=values.device, dtype=values.dtype + ) + values_b = torch.empty( + (total_len_b, D), device=values.device, dtype=values.dtype + ) + _triton_split_2D_jagged_internal( + jagged_in=values, + max_seq_len=max_seq_len, + B=B, + offsets_a=offsets_a, + offsets_b=offsets_b, + max_len_a=max_len_a, + max_len_b=max_len_b, + out_a=values_a, + out_b=values_b, + D=D, + n_prefix_to_B=n_prefix_to_B, + is_dense_a=is_dense_a, + is_dense_b=is_dense_b, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_a, offsets_b) + ctx.max_seq_len = max_seq_len + ctx.total_seq_len = total_seq_len + ctx.max_len_a = max_len_a + ctx.max_len_b = max_len_b + ctx.is_dense_a = is_dense_a + ctx.is_dense_b = is_dense_b + ctx.B = B + ctx.D = D + ctx.n_prefix_to_B = n_prefix_to_B + return values_a, values_b + + @staticmethod + def backward( + ctx, *d_values + ) -> Tuple[None, torch.Tensor, None, None, None, None, None, None, None]: + offsets_a, offsets_b = ctx.saved_tensors + d_values_a, d_values_b = d_values + BLOCK_D = triton.next_power_of_2(ctx.D) + d_jagged_in = torch.empty( + (ctx.total_seq_len, ctx.D), + device=d_values_a.device, + dtype=d_values_a.dtype, + ) + _triton_concat_2D_jagged_internal( + values_a=d_values_a, + values_b=d_values_b, + values_out=d_jagged_in, + max_seq_len=ctx.max_seq_len, + B=ctx.B, + offsets_a=offsets_a, + offsets_b=offsets_b, + max_len_a=ctx.max_len_a, + max_len_b=ctx.max_len_b, + D=ctx.D, + n_prefix_from_B=ctx.n_prefix_to_B, + is_dense_a=ctx.is_dense_a, + is_dense_b=ctx.is_dense_b, + BLOCK_D=BLOCK_D, + ) + + return None, d_jagged_in, None, None, None, None, None, None, None + + +@torch.jit.unused +@torch.fx.wrap +def triton_concat_2D_jagged( + max_seq_len: int, + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + n_prefix_from_right: int = 0, +) -> torch.Tensor: + return _Concat2DJaggedFunction.apply( + max_seq_len, + values_left, + values_right, + max_len_left, + max_len_right, + offsets_left, + offsets_right, + n_prefix_from_right, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_split_2D_jagged( + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int], + total_len_right: Optional[int], + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + n_prefix_to_right: int = 0, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _Split2DJaggedFunction.apply( + max_seq_len, + values, + total_len_left, + total_len_right, + max_len_left, + max_len_right, + offsets_left, + offsets_right, + n_prefix_to_right, + ) + + +class _Concat2DJaggedMultirowFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values_left: torch.Tensor, + values_right: torch.Tensor, + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + n_prefix_from_right: int, + ) -> torch.Tensor: + values_left = switch_to_contiguous_if_needed(values_left) + values_right = switch_to_contiguous_if_needed(values_right) + is_dense_left = offsets_left is None + is_dense_right = offsets_right is None + total_len_left, D = values_left.shape + total_len_right, _ = values_right.shape + if is_dense_left: + assert max_len_left is not None + B = total_len_left // max_len_left + else: + assert offsets_left is not None + B = offsets_left.shape[0] - 1 + if is_dense_right: + assert max_len_right is not None + B = total_len_right // max_len_right + else: + assert offsets_right is not None + B = offsets_right.shape[0] - 1 + total_seq_len = total_len_left + total_len_right + BLOCK_D = triton.next_power_of_2(D) + values_out = torch.empty( + (total_seq_len, D), device=values_left.device, dtype=values_left.dtype + ) + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + concat_2D_jagged_multirow[grid]( + ValuesA=values_left, + ValuesB=values_right, + OffsetsA=offsets_left, + OffsetsB=offsets_right, + MaxLenA=max_len_left, + MaxLenB=max_len_right, + Out=values_out, + D=D, + stride_ad=values_left.stride(-2), + stride_bd=values_right.stride(-2), + stride_od=values_out.stride(-2), + n_prefix_from_B=n_prefix_from_right, + IS_DENSE_A=is_dense_left, + IS_DENSE_B=is_dense_right, + BLOCK_D=BLOCK_D, + ) + ctx.save_for_backward(offsets_left, offsets_right) + ctx.max_seq_len = max_seq_len + ctx.total_len_left = total_len_left + ctx.total_len_right = total_len_right + ctx.is_dense_left = is_dense_left + ctx.is_dense_right = is_dense_right + ctx.max_len_left = max_len_left + ctx.max_len_right = max_len_right + ctx.B = B + ctx.n_prefix_from_right = n_prefix_from_right + return values_out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[None, torch.Tensor, torch.Tensor, None, None, None, None, None]: + offsets_left, offsets_right = ctx.saved_tensors + _, D = d_out.shape + BLOCK_D = triton.next_power_of_2(D) + d_values_left = torch.zeros( + (ctx.total_len_left, D), device=d_out.device, dtype=d_out.dtype + ) + d_values_right = torch.empty( + (ctx.total_len_right, D), device=d_out.device, dtype=d_out.dtype + ) + + def grid(meta): + return (triton.cdiv(ctx.max_seq_len, meta["BLOCK_N"]), ctx.B) + + split_2D_jagged_multirow[grid]( + JaggedIn=d_out, + OffsetsA=offsets_left, + OffsetsB=offsets_right, + MaxLenA=ctx.max_len_left, + MaxLenB=ctx.max_len_right, + OutA=d_values_left, + OutB=d_values_right, + D=D, + stride_id=d_out.stride(-2), + stride_ad=d_values_left.stride(-2), + stride_bd=d_values_right.stride(-2), + n_prefix_to_B=ctx.n_prefix_from_right, + IS_DENSE_A=ctx.is_dense_left, + IS_DENSE_B=ctx.is_dense_right, + BLOCK_D=BLOCK_D, + ) + return None, d_values_left, d_values_right, None, None, None, None, None + + +class _Split2DJaggedMultirowFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int], + total_len_right: Optional[int], + max_len_left: Optional[int], + max_len_right: Optional[int], + offsets_left: Optional[torch.Tensor], + offsets_right: Optional[torch.Tensor], + ) -> Tuple[torch.Tensor, torch.Tensor]: + values = switch_to_contiguous_if_needed(values) + is_dense_left: bool = offsets_left is None + is_dense_right: bool = offsets_right is None + total_seq_len, D = values.shape + + if is_dense_left: + assert is_dense_right is False + assert offsets_right is not None + assert max_len_left is not None + B = offsets_right.shape[0] - 1 + total_len_a = max_len_left * B + total_len_b = total_seq_len - total_len_a + elif is_dense_right: + assert is_dense_left is False + assert offsets_left is not None + assert max_len_right is not None + B = offsets_left.shape[0] - 1 + total_len_b = max_len_right * B + total_len_a = total_seq_len - total_len_b + else: + assert offsets_left is not None and offsets_right is not None + B = offsets_left.shape[0] - 1 + if total_len_left is not None and total_len_right is not None: + assert total_len_left + total_len_right == total_seq_len + total_len_a = total_len_left + total_len_b = total_len_right + else: + total_len_a = int(offsets_left[-1].item()) + total_len_b = values.size(0) - total_len_a + + BLOCK_D = triton.next_power_of_2(D) + values_a = torch.empty( + (total_len_a, D), device=values.device, dtype=values.dtype + ) + values_b = torch.empty( + (total_len_b, D), device=values.device, dtype=values.dtype + ) + + def grid(meta): + return (triton.cdiv(max_seq_len, meta["BLOCK_N"]), B) + + split_2D_jagged_multirow[grid]( + JaggedIn=values, + OffsetsA=offsets_left, + OffsetsB=offsets_right, + MaxLenA=max_len_left, + MaxLenB=max_len_right, + OutA=values_a, + OutB=values_b, + D=D, + stride_id=values.stride(-2), + stride_ad=values_a.stride(-2), + stride_bd=values_b.stride(-2), + n_prefix_to_B=0, + IS_DENSE_A=is_dense_left, + IS_DENSE_B=is_dense_right, + BLOCK_D=BLOCK_D, + ) + + ctx.save_for_backward(offsets_left, offsets_right) + ctx.max_seq_len = max_seq_len + ctx.total_seq_len = total_seq_len + ctx.max_len_left = max_len_left + ctx.max_len_right = max_len_right + ctx.is_dense_left = is_dense_left + ctx.is_dense_right = is_dense_right + ctx.B = B + ctx.D = D + + return values_a, values_b + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, *d_values + ) -> Tuple[None, torch.Tensor, None, None, None, None, None, None]: + offsets_left, offsets_right = ctx.saved_tensors + d_values_a, d_values_b = d_values + BLOCK_D = triton.next_power_of_2(ctx.D) + d_jagged_in = torch.empty( + (ctx.total_seq_len, ctx.D), + device=d_values_a.device, + dtype=d_values_a.dtype, + ) + + def grid(meta): + return (triton.cdiv(ctx.max_seq_len, meta["BLOCK_N"]), ctx.B) + + concat_2D_jagged_multirow[grid]( + ValuesA=d_values_a, + ValuesB=d_values_b, + OffsetsA=offsets_left, + OffsetsB=offsets_right, + MaxLenA=ctx.max_len_left, + MaxLenB=ctx.max_len_right, + Out=d_jagged_in, + D=ctx.D, + stride_ad=d_values_a.stride(-2), + stride_bd=d_values_b.stride(-2), + stride_od=d_jagged_in.stride(-2), + n_prefix_from_B=0, + IS_DENSE_A=ctx.is_dense_left, + IS_DENSE_B=ctx.is_dense_right, + BLOCK_D=BLOCK_D, + ) + + return None, d_jagged_in, None, None, None, None, None, None + + +@torch.jit.unused +@torch.fx.wrap +def triton_concat_2D_jagged_multirow( + max_seq_len: int, + values_a: torch.Tensor, + values_b: torch.Tensor, + offsets_a: Optional[torch.Tensor], + offsets_b: Optional[torch.Tensor], + max_len_a: int, + max_len_b: int, +) -> torch.Tensor: + return _Concat2DJaggedMultirowFunction.apply( + max_seq_len, + values_a, + values_b, + max_len_a, + max_len_b, + offsets_a, + offsets_b, + 0, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_split_2D_jagged_multirow( + max_seq_len: int, + values: torch.Tensor, + total_len_left: Optional[int] = None, + total_len_right: Optional[int] = None, + max_len_left: Optional[int] = None, + max_len_right: Optional[int] = None, + offsets_left: Optional[torch.Tensor] = None, + offsets_right: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor]: + return _Split2DJaggedMultirowFunction.apply( + max_seq_len, + values, + total_len_left, + total_len_right, + max_len_left, + max_len_right, + offsets_left, + offsets_right, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py new file mode 100644 index 000000000..cc513433b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_layer_norm.py @@ -0,0 +1,1337 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import ( + switch_to_contiguous_if_needed, + triton_autotune, +) +from generative_recommenders.ops.triton._autotune_pinning import pinned_or_full +from generative_recommenders.ops.utils import ( + is_sm100_plus, + is_sm90, + maybe_register_custom_op, +) + +try: + # @manual=//triton:triton + from triton.language.extra.libdevice import fast_dividef, rsqrt as libdevice_rsqrt +except ImportError: + try: + # @manual=//triton:triton + from triton.language.extra.cuda.libdevice import ( + fast_dividef, + rsqrt as libdevice_rsqrt, + ) + except ImportError: + # pyre-ignore: Undefined import [21] + # @manual=//triton:triton + from triton.language.math import fast_dividef, rsqrt as libdevice_rsqrt + + +def _get_layer_norm_fwd_configs() -> List[triton.Config]: + """Generate autotune configs for multi-row LayerNorm kernels.""" + configs = [] + block_ns = [4, 8, 16] if is_sm100_plus() else [1, 2, 4, 8] + for BLOCK_N in block_ns: + for num_warps in [1, 2, 4, 8]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +def _bwd_pre_hook(nargs): + nargs["DW"].zero_() + if "DB" in nargs: + nargs["DB"].zero_() + + +def _get_norm_bwd_configs() -> List[triton.Config]: + """Generate autotune configs for multi-row LayerNorm kernels.""" + configs = [] + if is_sm100_plus(): + block_ns = [8, 16] + num_shards_list = [8, 16] + num_warps_list = [2, 4] + elif is_sm90(): + block_ns = [2, 4] + num_shards_list = [8] + num_warps_list = [2, 4] + else: + block_ns = [1, 2] + num_shards_list = [8] + num_warps_list = [2, 4] + for BLOCK_N in block_ns: + for num_warps in num_warps_list: + for num_shards in num_shards_list: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N, "SHARDS_PER_SM": num_shards}, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_get_layer_norm_fwd_configs(), + key=["BLOCK_D"], +) +@triton.jit +def _layer_norm_fwd( + X, + Y, + Mean, + Rstd, + N, + D, + eps, + stride_x, + stride_y, + TRAINING: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + COMPUTE_MEAN_AND_RSTD: tl.constexpr, +): + block_id = tl.program_id(0) + start_row = block_id * BLOCK_N + + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr = tl.make_block_ptr( + base=Y, + shape=(N, D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + + if COMPUTE_MEAN_AND_RSTD: + mean = tl.sum(x_block, axis=1) / D + if TRAINING: + tl.store(Mean + rows, mean, row_mask) + mean = tl.expand_dims(mean, 1) + else: + mean = tl.load(Mean + rows, row_mask, other=0.0) + mean = tl.expand_dims(mean, 1) + + x_mean = x_block - mean + x_mean = tl.where(row_mask[:, None] & col_mask[None, :], x_mean, 0.0) + + if COMPUTE_MEAN_AND_RSTD: + _var = x_mean * x_mean + var = tl.sum(_var, axis=1) / D + rstd = 1 / tl.sqrt(var + eps) + if TRAINING: + tl.store(Rstd + rows, rstd, row_mask) + else: + rstd = tl.load(Rstd + rows, row_mask, other=0.0) + + rstd = tl.expand_dims(rstd, 1) + y = x_mean * rstd + + tl.store(Y_block_ptr, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton_autotune( + configs=_get_layer_norm_fwd_configs(), + key=["BLOCK_D"], +) +@triton.jit +def _weighted_layer_norm_fwd( + X, + Y, + W, + B, + Mean, + Rstd, + N, + D, + eps, + stride_x, + stride_y, + IS_SWISH: tl.constexpr, + TRAINING: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + COMPUTE_MEAN_AND_RSTD: tl.constexpr, +): + # Get the block ID and calculate starting row + block_id = tl.program_id(0) + start_row = block_id * BLOCK_N + + # Load weight and bias once (shared across all rows in this block) + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + w = tl.load(W + cols, mask=col_mask, other=0.0).to(tl.float32) + b = tl.load(B + cols, mask=col_mask, other=0.0).to(tl.float32) + + # Create block pointers for X and Y + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr = tl.make_block_ptr( + base=Y, + shape=(N, D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + + if COMPUTE_MEAN_AND_RSTD: + mean = tl.sum(x_block, axis=1) / D + if TRAINING: + tl.store(Mean + rows, mean, row_mask) + mean = tl.expand_dims(mean, 1) + else: + mean = tl.load(Mean + rows, row_mask, other=0.0) + mean = tl.expand_dims(mean, 1) + + x_mean = x_block - mean + x_mean = tl.where(row_mask[:, None] & col_mask[None, :], x_mean, 0.0) + + if COMPUTE_MEAN_AND_RSTD: + _var = x_mean * x_mean + var = tl.sum(_var, axis=1) / D + rstd = libdevice_rsqrt(var + eps) + if TRAINING: + tl.store(Rstd + rows, rstd, row_mask) + else: + rstd = tl.load(Rstd + rows, row_mask, other=0.0) + + rstd = tl.expand_dims(rstd, 1) + y = x_mean * rstd + y = y * w[None, :] + b[None, :] + + if IS_SWISH: + y = tl.sigmoid(y) * x_block + + tl.store(Y_block_ptr, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.jit +def _layer_norm_bwd_dx( + DX, + DY, + X, + Mean, + Rstd, + stride_dx, + stride_dy, + stride_x, + D, + eps, + BLOCK_D: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.arange(0, BLOCK_D) + mask = cols < D + X += row.to(tl.int64) * stride_x + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + + # Load data to SRAM + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + mean = tl.load(Mean + row) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = (x - mean) * rstd + xhat = tl.where(mask, xhat, 0.0) + dy = tl.where(mask, dy, 0.0) + c1 = tl.sum(xhat * dy, axis=0) / D + c2 = tl.sum(dy, axis=0) / D + dx = (dy - (xhat * c1 + c2)) * rstd + # Write dx + tl.store(DX + cols, dx, mask=mask) + + +@triton_autotune( + configs=pinned_or_full( + [ + triton.Config({"BLOCK_N": 1}, num_warps=1), # bs=32 winner + triton.Config({"BLOCK_N": 8}, num_warps=1), # bs=1024 winner + ], + _get_layer_norm_fwd_configs, + ), + key=["BLOCK_D"], +) +@triton.jit +def _weighted_layer_norm_bwd_dx( + DX, + DY, + DW, + DB, + X, + W, + B, + Mean, + Rstd, + stride_dx, + stride_dy, + stride_x, + D, + eps, + IS_SWISH: tl.constexpr, + N, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + num_blocks = tl.cdiv(N, BLOCK_N) + blocks_per_tile = num_blocks // tile_num + if pid < num_blocks % tile_num: + blocks_per_tile += 1 + + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + w = tl.load(W + cols, mask=col_mask, other=0.0).to(tl.float32) + + acc_dw = tl.zeros([BLOCK_D], dtype=tl.float32) + acc_db = tl.zeros([BLOCK_D], dtype=tl.float32) + + start_block = pid + + for idx in range(blocks_per_tile): + current_block = start_block + idx * tile_num + start_row = current_block * BLOCK_N + + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + DX_block_ptr = tl.make_block_ptr( + base=DX, + shape=(N, D), + strides=(stride_dx, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + DY_block_ptr = tl.make_block_ptr( + base=DY, + shape=(N, D), + strides=(stride_dy, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + # Load data blocks + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + dy_block = tl.load( + DY_block_ptr, boundary_check=(0, 1), padding_option="zero" + ).to(tl.float32) + + # Load mean and rstd for all rows in this block + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + mean = tl.load(Mean + rows, row_mask, other=0.0) + rstd = tl.load(Rstd + rows, row_mask, other=0.0) + + # Expand dimensions for broadcasting + mean = tl.expand_dims(mean, 1) + rstd = tl.expand_dims(rstd, 1) + + xhat = (x_block - mean) * rstd + + xhat = tl.where(row_mask[:, None] & col_mask[None, :], xhat, 0.0) + wdy = w[None, :] * dy_block + wdy = tl.where(row_mask[:, None] & col_mask[None, :], wdy, 0.0) + + # Compute dx + if IS_SWISH: + b = tl.load(B + cols, mask=col_mask, other=0.0).to(tl.float32) + sigmoid_layer_norm = tl.sigmoid(xhat * w[None, :] + b[None, :]) + sigmoid_layer_norm = tl.where( + row_mask[:, None] & col_mask[None, :], sigmoid_layer_norm, 0.0 + ) + + sigmoid_deriv = sigmoid_layer_norm * (1 - sigmoid_layer_norm) + x_ = wdy * x_block * sigmoid_deriv + x_ = tl.where(row_mask[:, None] & col_mask[None, :], x_, 0.0) + + c1 = tl.sum(xhat * x_, axis=1) / D + c2 = tl.sum(x_, axis=1) / D + c1 = tl.expand_dims(c1, 1) + c2 = tl.expand_dims(c2, 1) + dx = (x_ - (xhat * c1 + c2)) * rstd + + dx = dy_block * sigmoid_layer_norm + dx + # Write dx + tl.store(DX_block_ptr, dx.to(DX.dtype.element_ty), boundary_check=(0, 1)) + partial_dw = tl.sum(dy_block * x_block * xhat * sigmoid_deriv, axis=0) + partial_db = tl.sum(dy_block * x_block * sigmoid_deriv, axis=0) + else: + c1 = tl.sum(xhat * wdy, axis=1) / D + c2 = tl.sum(wdy, axis=1) / D + c1 = tl.expand_dims(c1, 1) + c2 = tl.expand_dims(c2, 1) + dx = (wdy - (xhat * c1 + c2)) * rstd + # Write dx + tl.store(DX_block_ptr, dx.to(DX.dtype.element_ty), boundary_check=(0, 1)) + partial_dw = tl.sum(dy_block * xhat, axis=0) + partial_db = tl.sum(dy_block, axis=0) + + # Accumulate partial sums in shared memory + acc_dw += partial_dw + acc_db += partial_db + + # Store accumulated sums back to global memory + dw_ptrs = DW + pid.to(tl.int64) * D + cols + db_ptrs = DB + pid.to(tl.int64) * D + cols + tl.store(dw_ptrs, acc_dw, mask=col_mask) + tl.store(db_ptrs, acc_db, mask=col_mask) + + +def _get_bwd_dwdb_configs() -> List[triton.Config]: + configs = [] + BLOCK_N_CHOICES = [32, 64, 128, 256] + if is_sm100_plus(): + BLOCK_N_CHOICES = [128, 256, 512, 1024] + for BLOCK_N in BLOCK_N_CHOICES: + for num_warps in [8, 16] + ([] if torch.ops.hip else [32]): + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=pinned_or_full( + [triton.Config({"BLOCK_N": 128}, num_warps=8)], + _get_bwd_dwdb_configs, + ), + key=["D"], +) +@triton.jit +def _layer_norm_bwd_dwdb( + DW, + DB, + FINAL_DW, + FINAL_DB, + N, + D, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid = tl.program_id(0) + cols = pid * BLOCK_D + tl.arange(0, BLOCK_D) + dw = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + db = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask = (rows[:, None] < N) & (cols[None, :] < D) + offs = rows[:, None] * D + cols[None, :] + dw += tl.load(DW + offs, mask=mask, other=0.0) + db += tl.load(DB + offs, mask=mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + sum_db = tl.sum(db, axis=0) + tl.store(FINAL_DW + cols, sum_dw.to(FINAL_DW.dtype.element_ty), mask=cols < D) + tl.store(FINAL_DB + cols, sum_db.to(FINAL_DB.dtype.element_ty), mask=cols < D) + + +def compute_BLOCK_D(x: torch.Tensor) -> int: + """Compute the BLOCK_D parameter for layer norm kernels.""" + D = x.shape[-1] + MAX_FUSED_SIZE = 65536 // x.element_size() + return min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + + +@maybe_register_custom_op( + "generative_recommenders::triton_weighted_layer_norm_fwd", mutates_args=() +) +def triton_weighted_layer_norm_fwd( + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, + mean: Optional[torch.Tensor] = None, + rstd: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + assert x.dim() == 2, f"x.dim() == {x.dim()}, expected 2" + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + learnable = weight is not None + if learnable: + assert bias is not None and weight is not None + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == D + assert bias.numel() == D + + y = torch.empty_like(x) + compute_mean_and_rstd = mean is None or rstd is None + # Always allocate new tensors to avoid aliasing inputs with outputs + out_mean = torch.empty((N,), dtype=torch.float32, device=x.device) + out_rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + if not compute_mean_and_rstd: + assert mean is not None and rstd is not None + out_mean.copy_(mean) + out_rstd.copy_(rstd) + + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D: int = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + if N == 0: + return y, out_mean, out_rstd + + # pyre-ignore[28] + grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),) # noqa E731 + if learnable: + _weighted_layer_norm_fwd[grid]( + x, + y, + weight, + bias, + out_mean, + out_rstd, + N, + D, + eps, + x.stride(0), + y.stride(0), + IS_SWISH=False, + TRAINING=True, + BLOCK_D=BLOCK_D, + COMPUTE_MEAN_AND_RSTD=compute_mean_and_rstd, + ) + else: + _layer_norm_fwd[grid]( + x, + y, + out_mean, + out_rstd, + N, + D, + eps, + x.stride(0), + y.stride(0), + TRAINING=True, + BLOCK_D=BLOCK_D, + COMPUTE_MEAN_AND_RSTD=compute_mean_and_rstd, + ) + + return y, out_mean, out_rstd + + +@triton_weighted_layer_norm_fwd.register_fake +def _( + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, + mean: Optional[torch.Tensor] = None, + rstd: Optional[torch.Tensor] = None, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + N = x.shape[0] + y = torch.empty_like(x) + # Always allocate new tensors to avoid aliasing inputs with outputs + out_mean = torch.empty((N,), dtype=torch.float32, device=x.device) + out_rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + return y, out_mean, out_rstd + + +@maybe_register_custom_op( + "generative_recommenders::triton_weighted_layer_norm_bwd", mutates_args=() +) +def _triton_weighted_layer_norm_bwd_impl( + dy: torch.Tensor, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + learnable: bool, + eps: float, + BLOCK_D: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + num_warps: int = min(max(BLOCK_D // 256, 1), 8) + if learnable: + N, D = x.shape + dx = torch.empty_like(x) + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + tile_num = max(1, min(sms * 8, N // 4)) + _dweight = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + _dbias = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + if N == 0: + dweight.zero_() + dbias.zero_() + return dx, dweight, dbias + # pyre-ignore[28] + _weighted_layer_norm_bwd_dx[(tile_num,)]( + dx, + dy, + _dweight, + _dbias, + x, + weight, + bias, + mean, + rstd, + dx.stride(0), + dy.stride(0), + x.stride(0), + D, + eps, + IS_SWISH=False, + N=N, + BLOCK_D=BLOCK_D, + ) + + def grid(META): + return (triton.cdiv(D, META["BLOCK_D"]),) + + blocks = triton.next_power_of_2(sms * 4) + BLOCK_D = triton.next_power_of_2(triton.cdiv(D, blocks)) + BLOCK_D = min(max(BLOCK_D, 4), 128) + _layer_norm_bwd_dwdb[grid]( + _dweight, + _dbias, + dweight, + dbias, + tile_num, + D, + BLOCK_D=BLOCK_D, + ) + + return dx, dweight, dbias + else: + N, D = x.shape + dx = torch.empty_like(x) + # Return empty tensors as sentinels for None + dweight = torch.empty(0, dtype=x.dtype, device=x.device) + dbias = torch.empty(0, dtype=x.dtype, device=x.device) + if N == 0: + return dx, dweight, dbias + # pyre-ignore[28] + _layer_norm_bwd_dx[(N,)]( + dx, + dy, + x, + mean, + rstd, + dx.stride(0), + dy.stride(0), + x.stride(0), + D, + eps, + BLOCK_D=BLOCK_D, + num_warps=num_warps, + ) + return dx, dweight, dbias + + +@_triton_weighted_layer_norm_bwd_impl.register_fake +def _( + dy: torch.Tensor, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + mean: torch.Tensor, + rstd: torch.Tensor, + learnable: bool, + eps: float, + BLOCK_D: int, +) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: + dx = torch.empty_like(x) + if learnable: + D = x.shape[-1] + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + else: + dweight = torch.empty(0, dtype=x.dtype, device=x.device) + dbias = torch.empty(0, dtype=x.dtype, device=x.device) + return dx, dweight, dbias + + +def triton_weighted_layer_norm_bwd( + dy: torch.Tensor, + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + mean: torch.Tensor, + rstd: torch.Tensor, + learnable: bool, + eps: float, + BLOCK_D: int, +) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor]]: + # Use sentinel tensors for custom_op compatibility (can't return Optional[Tensor]) + _weight = ( + weight if weight is not None else torch.empty(0, dtype=x.dtype, device=x.device) + ) + _bias = bias if bias is not None else torch.empty(0, dtype=x.dtype, device=x.device) + dx, dweight, dbias = _triton_weighted_layer_norm_bwd_impl( + dy=dy, + x=x, + weight=_weight, + bias=_bias, + mean=mean, + rstd=rstd, + learnable=learnable, + eps=eps, + BLOCK_D=BLOCK_D, + ) + if not learnable: + return dx, None, None + return dx, dweight, dbias + + +class LayerNormFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, + ) -> torch.Tensor: + y, mean, rstd = triton_weighted_layer_norm_fwd( + x=x, + weight=weight, + bias=bias, + eps=eps, + ) + BLOCK_D = compute_BLOCK_D(x) + learnable = weight is not None + if learnable: + ctx.save_for_backward(x, weight, bias, mean, rstd) + else: + ctx.save_for_backward(x, mean, rstd) + ctx.BLOCK_D = BLOCK_D + ctx.eps = eps + ctx.learnable = learnable + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], None]: + if ctx.learnable: + x, weight, bias, mean, rstd = ctx.saved_tensors + else: + x, mean, rstd = ctx.saved_tensors + weight, bias = None, None + dx, dweight, dbias = triton_weighted_layer_norm_bwd( + dy=dy, + x=x, + weight=weight, + bias=bias, + mean=mean, + rstd=rstd, + learnable=ctx.learnable, + eps=ctx.eps, + BLOCK_D=ctx.BLOCK_D, + ) + return dx, dweight, dbias, None + + +def _get_rms_norm_fwd_configs() -> List[triton.Config]: + """Generate autotune configs for multi-row RMSNorm kernels.""" + configs = [] + for BLOCK_N in [1, 4, 16]: + for num_warps in [2, 4]: + configs.append( + triton.Config( + {"BLOCK_N": BLOCK_N}, + num_warps=num_warps, + ) + ) + return configs + + +@triton.autotune( + configs=_get_rms_norm_fwd_configs(), + key=["BLOCK_D", "SILU"], +) +@triton.jit +def _weighted_rms_norm_fwd( + X, + Y, + W, + Rstd, + N, + D: tl.constexpr, + eps, + stride_x, + stride_y, + SILU: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + block_id = tl.program_id(0) + start_row = block_id * BLOCK_N + + # Load weight once (shared across all rows in this block) + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + w = tl.load(W + cols, mask=col_mask, other=0.0).to(tl.float32) + + # Create block pointers for X and Y + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + Y_block_ptr = tl.make_block_ptr( + base=Y, + shape=(N, D), + strides=(stride_y, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + + # Compute variance (RMS norm uses x directly, not x - mean) + x_masked = tl.where(row_mask[:, None] & col_mask[None, :], x_block, 0.0) + _var = x_masked * x_masked + var = tl.sum(_var, axis=1) / D + rstd = 1 / tl.sqrt(var + eps) + tl.store(Rstd + rows, rstd, row_mask) + + # Normalize and apply linear transformation + rstd = tl.expand_dims(rstd, 1) + y = x_block * rstd + y = y * w[None, :] + + if SILU: + # pyre-ignore[16]: Module `triton.language.math` has no attribute `fast_dividef` + y = fast_dividef(y, 1.0 + tl.exp(-y)) + + tl.store(Y_block_ptr, y.to(Y.dtype.element_ty), boundary_check=(0, 1)) + + +@triton.jit +def _weighted_rms_norm_bwd_dx( + DX, + DY, + DW, + X, + W, + Rstd, + Lock, + stride_dx, + stride_dy, + stride_x, + D: tl.constexpr, + eps, + GROUP_N, + BLOCK_D: tl.constexpr, +): + row = tl.program_id(0) + cols = tl.arange(0, BLOCK_D) + mask = cols < D + X += row.to(tl.int64) * stride_x + DY += row.to(tl.int64) * stride_dy + DX += row.to(tl.int64) * stride_dx + + # Load data to SRAM + x = tl.load(X + cols, mask=mask, other=0).to(tl.float32) + dy = tl.load(DY + cols, mask=mask, other=0).to(tl.float32) + rstd = tl.load(Rstd + row) + + # Compute dx + xhat = x * rstd + w = tl.load(W + cols, mask=mask).to(tl.float32) + wdy = w * dy + + xhat = tl.where(mask, xhat, 0.0) + wdy = tl.where(mask, wdy, 0.0) + c1 = tl.sum(xhat * wdy, axis=0) / D + dx = (wdy - (xhat * c1)) * rstd + # Write dx + tl.store(DX + cols, dx, mask=mask) + + # Offset locks and weights/biases gradient pointer for parallel reduction + lock_id = row % GROUP_N + Lock += lock_id + Count = Lock + GROUP_N + DW = DW + lock_id * D + cols + # Accumulate partial sums for dw/db + partial_dw = dy * xhat + while tl.atomic_cas(Lock, 0, 1) == 1: + pass + count = tl.load(Count) + # First store doesn't accumulate + if count == 0: + tl.atomic_xchg(Count, 1) + else: + partial_dw += tl.load(DW, mask=mask) + tl.store(DW, partial_dw, mask=mask) + # Release the lock + tl.atomic_xchg(Lock, 0) + + +@triton_autotune( + configs=_get_norm_bwd_configs(), + key=["BLOCK_D", "SILU"], + reset_to_zero=["DW"], +) +@triton.jit +def _weighted_rms_norm_bwd( + DX, + DY, + DW, + X, + W, + Rstd, + stride_dx, + stride_dy, + stride_x, + D, + eps, + N, + SILU: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, + SHARDS_PER_SM: tl.constexpr, +): + pid = tl.program_id(0) + tile_num = tl.num_programs(0) + num_blocks = tl.cdiv(N, BLOCK_N) + blocks_per_tile = num_blocks // tile_num + if pid < num_blocks % tile_num: + blocks_per_tile += 1 + + cols = tl.arange(0, BLOCK_D) + col_mask = cols < D + w = tl.load(W + cols, mask=col_mask, other=0.0).to(tl.float32) + + start_block = pid + + acc_dw = tl.zeros([BLOCK_D], dtype=tl.float32) + + for idx in range(blocks_per_tile): + current_block = start_block + idx * tile_num + start_row = current_block * BLOCK_N + + X_block_ptr = tl.make_block_ptr( + base=X, + shape=(N, D), + strides=(stride_x, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + DX_block_ptr = tl.make_block_ptr( + base=DX, + shape=(N, D), + strides=(stride_dx, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + DY_block_ptr = tl.make_block_ptr( + base=DY, + shape=(N, D), + strides=(stride_dy, 1), + offsets=(start_row, 0), + block_shape=(BLOCK_N, BLOCK_D), + order=(1, 0), + ) + + # Load data blocks + x_block = tl.load(X_block_ptr, boundary_check=(0, 1), padding_option="zero").to( + tl.float32 + ) + dy_block = tl.load( + DY_block_ptr, boundary_check=(0, 1), padding_option="zero" + ).to(tl.float32) + + # Load rstd for all rows in this block + rows = start_row + tl.arange(0, BLOCK_N) + row_mask = rows < N + rstd = tl.load(Rstd + rows, row_mask, other=0.0) + + # Expand dimensions for broadcasting + rstd = tl.expand_dims(rstd, 1) + + # Compute dx + xhat = x_block * rstd + + # Apply SILU backward if enabled + if SILU: + y_before_silu = xhat * w[None, :] + # pyre-fixme[16] + sig_y = fast_dividef(1.0, 1.0 + tl.exp(-y_before_silu)) + # SILU derivative: sigmoid(y) + y * sigmoid(y) * (1 - sigmoid(y)) + dy_block = dy_block * (sig_y + y_before_silu * sig_y * (1.0 - sig_y)) + + wdy = w[None, :] * dy_block + + c1 = tl.sum(xhat * wdy, axis=1) / D + c1 = tl.expand_dims(c1, 1) + dx = (wdy - (xhat * c1)) * rstd + + # Write dx + tl.store(DX_block_ptr, dx.to(DX.dtype.element_ty), boundary_check=(0, 1)) + + # Accumulate partial sums for dw + # Compute dw for all rows, then sum locally before atomic operation + partial_dw_block = dy_block * xhat + # Local reduction: sum across all rows in this block + partial_dw = tl.sum(partial_dw_block, axis=0) + acc_dw += partial_dw + + DW_ptr = DW + cols + tl.atomic_add(DW_ptr, acc_dw, col_mask) + + +@triton_autotune( + configs=_get_bwd_dwdb_configs(), + key=["D"], +) +@triton.jit +def _rms_norm_bwd_dwdb( + DW, + FINAL_DW, + N, + D, + BLOCK_N: tl.constexpr, + BLOCK_D: tl.constexpr, +): + pid = tl.program_id(0) + cols = pid * BLOCK_D + tl.arange(0, BLOCK_D) + dw = tl.zeros((BLOCK_N, BLOCK_D), dtype=tl.float32) + + for i in range(0, N, BLOCK_N): + rows = i + tl.arange(0, BLOCK_N) + # pyre-fixme[16]: `int` has no attribute `__getitem__`. + mask = (rows[:, None] < N) & (cols[None, :] < D) + offs = rows[:, None] * D + cols[None, :] + dw += tl.load(DW + offs, mask=mask, other=0.0) + + sum_dw = tl.sum(dw, axis=0) + tl.store(FINAL_DW + cols, sum_dw.to(FINAL_DW.dtype.element_ty), mask=cols < D) + + +class RMSNormFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + weight: torch.Tensor, + eps: float, + silu: bool, + ) -> torch.Tensor: + assert x.dim() == 2 + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + assert weight.dim() == 1 + assert weight.numel() == D + + y = torch.empty_like(x) + rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + + # Less than 64KB per feature: enqueue fused kernel + MAX_FUSED_SIZE = 65536 // x.element_size() + BLOCK_D = min(MAX_FUSED_SIZE, triton.next_power_of_2(D)) + if D > BLOCK_D: + raise RuntimeError("This layer norm doesn't support feature dim >= 64KB.") + + ctx.save_for_backward(x, weight, rstd) + ctx.silu = silu + if N == 0: + return y + + # pyre-ignore[28] + grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),) # noqa E731 + _weighted_rms_norm_fwd[grid]( + x, + y, + weight, + rstd, + N, + D, + eps, + x.stride(0), + y.stride(0), + SILU=silu, + BLOCK_D=BLOCK_D, + ) + + ctx.BLOCK_D = BLOCK_D + ctx.eps = eps + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], None, None]: + x, weight, rstd = ctx.saved_tensors + N, D = x.shape + dx = torch.empty_like(x) + dweight = torch.zeros((D,), dtype=weight.dtype, device=x.device) + if N == 0: + dweight.zero_() + return dx, dweight, None, None + + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + + # pyre-ignore[28] + grid = lambda meta: ( # noqa E731 + max(1, min(sms * meta["SHARDS_PER_SM"], N // 4)), + ) + _weighted_rms_norm_bwd[grid]( + dx, + dy, + dweight, + x, + weight, + rstd, + dx.stride(0), + dy.stride(0), + x.stride(0), + D, + ctx.eps, + N=N, + SILU=ctx.silu, + BLOCK_D=ctx.BLOCK_D, + ) + + return dx, dweight, None, None + + +class SwishLayerNormFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + x: torch.Tensor, + weight: torch.Tensor, + bias: torch.Tensor, + eps: float, + ) -> torch.Tensor: + assert x.dim() == 2, f"x.dim() == {x.dim()}, expected 2" + x = switch_to_contiguous_if_needed(x) + N, D = x.shape + + assert bias is not None and weight is not None + assert weight.dim() == 1 + assert bias.dim() == 1 + assert weight.numel() == D + assert bias.numel() == D + + y = torch.empty_like(x) + mean = torch.empty((N,), dtype=torch.float32, device=x.device) + rstd = torch.empty((N,), dtype=torch.float32, device=x.device) + + BLOCK_D = triton.next_power_of_2(D) + num_warps = min(max(BLOCK_D // 256, 1), 8) + + ctx.save_for_backward(x, weight, bias, mean, rstd) + ctx.BLOCK_D = BLOCK_D + ctx.num_warps = num_warps + ctx.eps = eps + if N == 0: + return y + + # pyre-ignore[28] + grid = lambda meta: (triton.cdiv(N, meta["BLOCK_N"]),) # noqa E731 + _weighted_layer_norm_fwd[grid]( + x, + y, + weight, + bias, + mean, + rstd, + N, + D, + eps, + x.stride(0), + y.stride(0), + IS_SWISH=True, + TRAINING=True, + BLOCK_D=BLOCK_D, + COMPUTE_MEAN_AND_RSTD=True, + ) + + return y + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, dy: torch.Tensor + ) -> Tuple[torch.Tensor, Optional[torch.Tensor], Optional[torch.Tensor], None]: + x, weight, bias, mean, rstd = ctx.saved_tensors + N, D = x.shape + dx = torch.empty_like(x) + sms = torch.cuda.get_device_properties(x.device).multi_processor_count + tile_num = max(1, min(sms * 8, N // 4)) + _dweight = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + _dbias = torch.empty((tile_num, D), dtype=torch.float32, device=x.device) + dweight = torch.empty((D,), dtype=weight.dtype, device=x.device) + dbias = torch.empty((D,), dtype=weight.dtype, device=x.device) + if N == 0: + dweight.zero_() + dbias.zero_() + return dx, dweight, dbias, None + # pyre-ignore[28] + _weighted_layer_norm_bwd_dx[(tile_num,)]( + dx, + dy, + _dweight, + _dbias, + x, + weight, + bias, + mean, + rstd, + dx.stride(0), + dy.stride(0), + x.stride(0), + D, + ctx.eps, + IS_SWISH=True, + N=N, + BLOCK_D=ctx.BLOCK_D, + ) + + def grid(META): + return (triton.cdiv(D, META["BLOCK_D"]),) + + blocks = triton.next_power_of_2(sms * 4) + BLOCK_D = triton.next_power_of_2(triton.cdiv(D, blocks)) + BLOCK_D = min(max(BLOCK_D, 4), 128) + _layer_norm_bwd_dwdb[grid]( + _dweight, + _dbias, + dweight, + dbias, + tile_num, + D, + BLOCK_D=BLOCK_D, + ) + + return dx, dweight, dbias, None + + +@torch.jit.unused +@torch.fx.wrap +def triton_layer_norm( + x: torch.Tensor, + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, +) -> torch.Tensor: + return LayerNormFunction.apply(x, weight, bias, eps) + + +@torch.jit.unused +@torch.fx.wrap +def triton_rms_norm( + x: torch.Tensor, + weight: Optional[torch.Tensor], + eps: float, + silu: bool = False, +) -> torch.Tensor: + return RMSNormFunction.apply(x, weight, eps, silu) + + +@torch.jit.unused +@torch.fx.wrap +def triton_swish_layer_norm( + x: torch.Tensor, + normalized_shape: List[int], + weight: Optional[torch.Tensor], + bias: Optional[torch.Tensor], + eps: float, +) -> torch.Tensor: + return SwishLayerNormFunction.apply(x, weight, bias, eps) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_position.py b/recommendation_v4/generative_recommenders/ops/triton/triton_position.py new file mode 100644 index 000000000..72c43f9ac --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_position.py @@ -0,0 +1,438 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +#!/usr/bin/env python3 + + +from typing import List, Optional, Tuple + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl + +try: + torch.ops.load_library("//hammer/ops/cuda:cuda_ops") +except OSError: + pass + +from generative_recommenders.common import ( + autotune_max_seq_len, + prev_power_of_2, + switch_to_contiguous_if_needed, + triton_autotune, +) + + +def _autotune_configs() -> List[triton.Config]: + configs = [] + for BLOCK_N in [16, 32, 64]: + for num_stages in [1, 2]: + for num_warps in [2, 4, 8]: + configs.append( + triton.Config( + { + "BLOCK_N": BLOCK_N, + }, + num_stages=num_stages, + num_warps=num_warps, + ) + ) + return configs + + +@triton_autotune( + configs=_autotune_configs(), + key=["AUTOTUNE_MAX_SEQ_LEN"], +) +@triton.jit +def _add_timestamp_position_embeddings_kernel( + SeqEmb, + Offsets, + Lengths, + PosEmb, + TsEmb, + Out, + TS, + PosInds, + TsInds, + NumTargets, + AUTOTUNE_MAX_SEQ_LEN, + D, + num_time_buckets, + time_bucket_increments, + time_bucket_scale, + time_delta, + max_contextual_seq_len, + max_pos_ind, + stride_sn, + stride_pn, + stride_tn, + stride_on, + TRAINING: tl.constexpr, + HAS_MULTIPLE_TARGETS: tl.constexpr, + INTERLEAVE_TARGETS: tl.constexpr, + TIME_BUCKET_FN: tl.constexpr, + BLOCK_D: tl.constexpr, + BLOCK_N: tl.constexpr, +): + """ + SeqEmb has shape (sum_B(N_i), D), + PosEmb has shape (N_p, D), + TsEmb has shape (N_t, D), + Out has shape (sum_B(N_i), D) + """ + + off_b = tl.program_id(0) + off_n = tl.program_id(1) + seq_start = tl.load(Offsets + off_b) + seq_end = tl.load(Offsets + off_b + 1) + seq_len = seq_end - seq_start + start_n = off_n * BLOCK_N + if start_n >= seq_len: + return + offs_n = start_n + tl.arange(0, BLOCK_N) + offs_d = tl.arange(0, BLOCK_D) + seq_emb_offsets = offs_n[:, None] * stride_sn + offs_d[None, :] + SeqEmb += seq_start.to(tl.int64) * stride_sn + mask_n = offs_n < seq_len + # position encoding + seq_len = tl.load(Lengths + off_b) + if HAS_MULTIPLE_TARGETS: + num_targets = tl.load(NumTargets + off_b) + if INTERLEAVE_TARGETS: + high_ind = seq_len - num_targets * 2 + else: + high_ind = seq_len - num_targets + else: + high_ind = seq_len + pos_inds = tl.where(offs_n < high_ind, offs_n, high_ind) + pos_inds = high_ind - pos_inds + max_contextual_seq_len + pos_inds = tl.where(pos_inds < max_pos_ind - 1, pos_inds, max_pos_ind - 1) + pos_inds = tl.where(offs_n < max_contextual_seq_len, offs_n, pos_inds) + if TRAINING: + tl.store(PosInds + seq_start + offs_n, pos_inds, mask=mask_n) + pos_emb_offsets = pos_inds[:, None] * stride_pn + offs_d[None, :] + # timestamp encoding + ts = tl.load(TS + seq_start + offs_n, mask=mask_n) + query_time = tl.load(TS + seq_end - 1) + ts = query_time - ts + time_delta + ts = tl.where(ts > 1e-6, ts, 1e-6) / time_bucket_increments + if TIME_BUCKET_FN == "log": + ts = tl.log(ts) + else: + ts = tl.sqrt(ts) + ts = ts * time_bucket_scale + ts = ts.to(tl.int32) + ts = tl.where(ts > 0, ts, 0) + ts = tl.where(ts < num_time_buckets, ts, num_time_buckets) + if TRAINING: + tl.store(TsInds + seq_start + offs_n, ts, mask=mask_n) + ts_emb_offsets = ts[:, None] * stride_tn + offs_d[None, :] + Out += seq_start.to(tl.int64) * stride_on + out_offsets = Out + offs_n[:, None] * stride_on + offs_d[None, :] + for _d in range(0, D, BLOCK_D): + mask = (offs_n[:, None] < seq_len) and offs_d[None, :] < D + seq_emb = tl.load(SeqEmb + seq_emb_offsets, mask=mask) + pos_emb = tl.load(PosEmb + pos_emb_offsets, mask=mask) + ts_emb = tl.load(TsEmb + ts_emb_offsets, mask=mask) + tl.store(out_offsets, seq_emb + (pos_emb + ts_emb).to(seq_emb.dtype), mask=mask) + seq_emb_offsets += BLOCK_D + pos_emb_offsets += BLOCK_D + ts_emb_offsets += BLOCK_D + out_offsets += BLOCK_D + offs_d += BLOCK_D + + +def bwd_pre_hook(nargs): + nargs["Out"].zero_() + + +def _add_embeddings_bwd_configs() -> List[triton.Config]: + configs = [] + for BLOCK in [32, 64, 128]: + for num_stages in [2, 3, 4]: + for num_warps in [2, 4, 8]: + configs.append( + triton.Config( + { + "BLOCK": BLOCK, + }, + num_stages=num_stages, + num_warps=num_warps, + pre_hook=bwd_pre_hook, + ) + ) + return configs + + +@triton_autotune( + configs=_add_embeddings_bwd_configs(), + key=["AUTOTUNE_MAX_SEQ_LEN", "AUTOTUNE_B", "D"], +) +@triton.jit +def _add_embeddings_bwd_kernel( + In, + KeyInds, + ValueInds, + Out, + AUTOTUNE_MAX_SEQ_LEN, + AUTOTUNE_B, + D, + jagged_size, + stride_in, + stride_on, + BLOCK_D: tl.constexpr, + BLOCK: tl.constexpr, +): + off_block = tl.program_id(0) + offs_d = tl.arange(0, BLOCK_D) + mask_d = offs_d < D + key_ind = -1 + key_ind = key_ind.to(KeyInds.dtype.element_ty) # pyre-ignore[16] + accumulator = tl.zeros((BLOCK_D,), dtype=In.dtype.element_ty) + for off_i in range(0, BLOCK): + off = off_block * BLOCK + off_i + if off < jagged_size: + value_ind = tl.load(ValueInds + off) + in_offset = In + value_ind.to(tl.int64) * stride_in + jagged_in = tl.load(in_offset + offs_d, mask=mask_d) + key_ind_new = tl.load(KeyInds + off) + if key_ind == key_ind_new: + accumulator += jagged_in + else: + if key_ind >= 0: + out_offset = Out + key_ind.to(tl.int64) * stride_on + tl.atomic_add( + out_offset + offs_d, + accumulator.to(Out.dtype.element_ty), + mask=mask_d, + sem="relaxed", + ) + key_ind = key_ind_new + accumulator = jagged_in + if key_ind >= 0: + out_offset = Out + key_ind.to(tl.int64) * stride_on + tl.atomic_add( + out_offset + offs_d, + accumulator.to(Out.dtype.element_ty), + mask=mask_d, + sem="relaxed", + ) + + +class _AddTimestampPositionEmbeddingsFunction(torch.autograd.Function): + @staticmethod + # pyre-ignore[14] + def forward( + ctx, + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + pos_embeddings: torch.Tensor, + ts_embeddings: torch.Tensor, + timestamps: torch.Tensor, + max_seq_len: int, + max_contextual_seq_len: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str, + ): + seq_embeddings = switch_to_contiguous_if_needed(seq_embeddings) + pos_embeddings = switch_to_contiguous_if_needed(pos_embeddings) + ts_embeddings = switch_to_contiguous_if_needed(ts_embeddings) + + max_pos_ind = pos_embeddings.shape[0] + B = seq_lengths.shape[0] + N, D = seq_embeddings.shape + assert len(pos_embeddings.shape) == 2 + assert len(ts_embeddings.shape) == 2 + assert pos_embeddings.shape[1] == D, ( + "shape[1] of pos_embeddings much match seq_embeddings" + ) + assert ts_embeddings.shape[1] == D, ( + "shape[1] of ts_embeddings much match seq_embeddings" + ) + out = torch.empty_like(seq_embeddings) + + timestamps = switch_to_contiguous_if_needed(timestamps) + ts_inds = torch.empty_like(seq_embeddings[:, 0], dtype=torch.int32) + pos_inds = torch.empty_like(seq_embeddings[:, 0], dtype=torch.int32) + ts_emb_size = ts_embeddings.shape[0] + + grid = lambda meta: ( # noqa E731 + B, + triton.cdiv(max_seq_len, meta["BLOCK_N"]), + ) + BLOCK_D = triton.next_power_of_2(D) if D < 64 else 64 + _add_timestamp_position_embeddings_kernel[grid]( + SeqEmb=seq_embeddings, + Offsets=seq_offsets, + Lengths=seq_lengths, + PosEmb=pos_embeddings, + TsEmb=ts_embeddings, + Out=out, + TS=timestamps, + PosInds=pos_inds, + TsInds=ts_inds, + NumTargets=num_targets, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(max_seq_len), + D=D, + num_time_buckets=ts_emb_size - 1, + time_bucket_increments=60.0, + time_bucket_scale=1.0, + time_delta=0, + max_contextual_seq_len=max_contextual_seq_len, + max_pos_ind=max_pos_ind, + stride_sn=seq_embeddings.stride(0), + stride_pn=pos_embeddings.stride(0), + stride_tn=ts_embeddings.stride(0), + stride_on=out.stride(0), + TRAINING=True, + HAS_MULTIPLE_TARGETS=num_targets is not None, + INTERLEAVE_TARGETS=interleave_targets, + TIME_BUCKET_FN=time_bucket_fn, + BLOCK_D=BLOCK_D, + ) + try: + values = torch.arange(0, N, dtype=torch.int32, device=timestamps.device) + sorted_ts_key_inds, sorted_ts_value_inds = torch.ops.hammer.sort_kv_pairs( + ts_inds, values + ) + sorted_pos_key_inds, sorted_pos_value_inds = torch.ops.hammer.sort_kv_pairs( + pos_inds, values + ) + except Exception: + sorted_ts_key_inds, sorted_ts_value_inds = torch.sort(ts_inds) + sorted_pos_key_inds, sorted_pos_value_inds = torch.sort(pos_inds) + ctx.save_for_backward( + sorted_pos_key_inds, + sorted_pos_value_inds, + sorted_ts_key_inds, + sorted_ts_value_inds, + ) + ctx.B = B + ctx.D = D + ctx.max_seq_len = max_seq_len + ctx.pos_emb_size = pos_embeddings.shape[0] + ctx.ts_emb_size = ts_emb_size + ctx.pos_dtype = pos_embeddings.dtype + ctx.ts_dtype = ts_embeddings.dtype + return out + + @staticmethod + # pyre-ignore[14] + def backward( + ctx, d_out: torch.Tensor + ) -> Tuple[ + torch.Tensor, + None, + torch.Tensor, + torch.Tensor, + None, + None, + None, + None, + None, + None, + None, + ]: + ( + sorted_pos_key_inds, + sorted_pos_value_inds, + sorted_ts_key_inds, + sorted_ts_value_inds, + ) = ctx.saved_tensors + d_pos_embeddings = torch.empty( + (ctx.pos_emb_size, ctx.D), device=d_out.device, dtype=torch.float32 + ) + d_ts_embeddings = torch.empty( + (ctx.ts_emb_size, ctx.D), device=d_out.device, dtype=torch.float32 + ) + grid = lambda meta: (triton.cdiv(d_out.shape[0], meta["BLOCK"]),) # noqa E731 + AUTOTUNE_B = prev_power_of_2(ctx.B) + _add_embeddings_bwd_kernel[grid]( + In=d_out, + KeyInds=sorted_pos_key_inds, + ValueInds=sorted_pos_value_inds, + Out=d_pos_embeddings, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(ctx.max_seq_len), + AUTOTUNE_B=AUTOTUNE_B, + D=ctx.D, + jagged_size=d_out.shape[0], + stride_in=d_out.stride(0), + stride_on=d_pos_embeddings.stride(0), + BLOCK_D=triton.next_power_of_2(ctx.D), + ) + _add_embeddings_bwd_kernel[grid]( + In=d_out, + KeyInds=sorted_ts_key_inds, + ValueInds=sorted_ts_value_inds, + Out=d_ts_embeddings, + AUTOTUNE_MAX_SEQ_LEN=autotune_max_seq_len(ctx.max_seq_len), + AUTOTUNE_B=AUTOTUNE_B, + D=ctx.D, + jagged_size=d_out.shape[0], + stride_in=d_out.stride(0), + stride_on=d_ts_embeddings.stride(0), + BLOCK_D=triton.next_power_of_2(ctx.D), + ) + return ( + d_out, + None, + d_pos_embeddings.to(ctx.pos_dtype), + d_ts_embeddings.to(ctx.ts_dtype), + None, + None, + None, + None, + None, + None, + None, + ) + + +@torch.jit.unused +@torch.fx.wrap +def triton_add_timestamp_positional_embeddings( + seq_embeddings: torch.Tensor, + seq_offsets: torch.Tensor, + pos_embeddings: torch.Tensor, + ts_embeddings: torch.Tensor, + timestamps: torch.Tensor, + max_seq_len: int, + max_contextual_seq_len: int, + seq_lengths: torch.Tensor, + num_targets: Optional[torch.Tensor], + interleave_targets: bool, + time_bucket_fn: str, +) -> torch.Tensor: + return _AddTimestampPositionEmbeddingsFunction.apply( + seq_embeddings, + seq_offsets, + pos_embeddings, + ts_embeddings, + timestamps, + max_seq_len, + max_contextual_seq_len, + seq_lengths, + num_targets, + interleave_targets, + time_bucket_fn, + ) diff --git a/recommendation_v4/generative_recommenders/ops/triton/triton_swiglu.py b/recommendation_v4/generative_recommenders/ops/triton/triton_swiglu.py new file mode 100644 index 000000000..5f30da53b --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/triton/triton_swiglu.py @@ -0,0 +1,753 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-unsafe + +from typing import List + +import torch + +# @manual=//triton:triton +import triton + +# @manual=//triton:triton +import triton.language as tl +from generative_recommenders.common import triton_autotune +from generative_recommenders.ops.utils import is_sm100_plus + +TMA_AVAILABLE = False +try: + # @manual=//triton:triton + from triton.tools.tensor_descriptor import TensorDescriptor + + TMA_AVAILABLE = True +except ImportError: + pass + +HAS_TLX = False +try: + # @manual=//triton:triton + import triton.language.extra.tlx as tlx # type: ignore + + HAS_TLX = True +except ImportError: + pass + + +def is_blackwell_triton_swiglu_supported() -> bool: + return is_sm100_plus() and TMA_AVAILABLE and HAS_TLX + + +def _swiglu_tma_set_block_size_hook(nargs) -> None: + BLOCK_M = nargs["BLOCK_M"] + BLOCK_N = nargs["BLOCK_N"] + BLOCK_K = nargs["BLOCK_K"] + EPILOGUE_SUBTILE = nargs.get("EPILOGUE_SUBTILE", 1) + + nargs["x_desc"].block_shape = [BLOCK_M, BLOCK_K] + nargs["w_gate_desc"].block_shape = [BLOCK_N, BLOCK_K] + nargs["w_up_desc"].block_shape = [BLOCK_N, BLOCK_K] + nargs["out_desc"].block_shape = [BLOCK_M, BLOCK_N // EPILOGUE_SUBTILE] + + +def get_swiglu_configs(pre_hook) -> List[triton.Config]: + return [ + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 128, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=1, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 64, + "BLOCK_N": 128, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=1, + num_warps=4, + pre_hook=pre_hook, + ), + triton.Config( + { + "BLOCK_M": 128, + "BLOCK_N": 64, + "BLOCK_K": 64, + "GROUP_M": 8, + }, + num_stages=1, + num_warps=4, + pre_hook=pre_hook, + ), + ] + + +@triton.jit +def _compute_pid_swiglu( + tile_id, + num_pid_in_group, + num_pid_m, + GROUP_M: tl.constexpr, + NUM_SMS: tl.constexpr, +): + group_id = tile_id // num_pid_in_group + first_pid_m = group_id * GROUP_M + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_m = first_pid_m + (tile_id % group_size_m) + pid_n = (tile_id % num_pid_in_group) // group_size_m + return pid_m, pid_n + + +@triton_autotune( + configs=get_swiglu_configs(pre_hook=_swiglu_tma_set_block_size_hook), + key=["M_BLOCK", "N", "K"], +) +@triton.jit +def _swiglu_fwd_tma_ws_persistent( + x_desc, + w_gate_desc, + w_up_desc, + out_desc, + M, + N, + K, + M_BLOCK, + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + NUM_SMEM_BUFFERS: tl.constexpr, + NUM_TMEM_BUFFERS: tl.constexpr, + NUM_SMS: tl.constexpr, + EPILOGUE_SUBTILE: tl.constexpr, +): + # Allocate SMEM buffers + x_buffers = tlx.local_alloc((BLOCK_M, BLOCK_K), x_desc.dtype, NUM_SMEM_BUFFERS) + + # Allocate SMEM buffers for W_gate and W_up + w_gate_buffers = tlx.local_alloc( + (BLOCK_N, BLOCK_K), w_gate_desc.dtype, NUM_SMEM_BUFFERS + ) + w_up_buffers = tlx.local_alloc( + (BLOCK_N, BLOCK_K), w_up_desc.dtype, NUM_SMEM_BUFFERS + ) + + # Allocate TMEM for accumulators + tmem_gate_buffers = tlx.local_alloc( + (BLOCK_M, BLOCK_N), tl.float32, NUM_TMEM_BUFFERS, tlx.storage_kind.tmem + ) + tmem_up_buffers = tlx.local_alloc( + (BLOCK_M, BLOCK_N), tl.float32, NUM_TMEM_BUFFERS, tlx.storage_kind.tmem + ) + + # Barriers for Producer <-> MMA synchronization + smem_full_bars_x_gate = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS, + arrive_count=1, # pyre-ignore[6] + ) + smem_full_bars_up = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS, + arrive_count=1, # pyre-ignore[6] + ) + # Empty barriers: arrive_count=2 because both GEMM1 and GEMM2 signal completion + smem_empty_bars = tlx.alloc_barriers( + num_barriers=NUM_SMEM_BUFFERS, + arrive_count=2, # pyre-ignore[6] + ) + + # Barriers for MMA <-> Epilogue synchronization + # pyre-ignore[6] + tmem_full_bars = tlx.alloc_barriers(num_barriers=NUM_TMEM_BUFFERS, arrive_count=1) + # pyre-ignore[6] + tmem_empty_bars = tlx.alloc_barriers(num_barriers=NUM_TMEM_BUFFERS, arrive_count=1) + + with tlx.async_tasks(): + # Epilogue Consumer: Reads from TMEM, applies SwiGLU, and stores to output + with tlx.async_task("default"): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + + # Initialize buffer tracking + processed_tiles = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid_swiglu( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_m = pid_m * BLOCK_M + offs_n = pid_n * BLOCK_N + + cur_tmem_buf = processed_tiles % int(NUM_TMEM_BUFFERS) + tmem_read_phase = (processed_tiles // int(NUM_TMEM_BUFFERS)) & 1 + + # Wait for MMA to finish writing to TMEM + # pyre-ignore[16] + tlx.barrier_wait(tmem_full_bars[cur_tmem_buf], tmem_read_phase) + + # Load gate and up results from TMEM + # pyre-ignore[16] + gate_tmem = tmem_gate_buffers[cur_tmem_buf] + up_tmem = tmem_up_buffers[cur_tmem_buf] + + if EPILOGUE_SUBTILE > 1: + # Process tile in subtiles + slice_size: tl.constexpr = BLOCK_N // EPILOGUE_SUBTILE + for slice_id in tl.static_range(EPILOGUE_SUBTILE): + gate_subslice = tlx.local_slice( + gate_tmem, + [0, slice_id * slice_size], + # pyre-ignore[6] + [BLOCK_M, slice_size], + ) + up_subslice = tlx.local_slice( + up_tmem, + [0, slice_id * slice_size], + # pyre-ignore[6] + [BLOCK_M, slice_size], + ) + + gate = tlx.local_load(gate_subslice).to(out_desc.dtype) + up = tlx.local_load(up_subslice).to(out_desc.dtype) + + gate_fp32 = gate.to(tl.float32) + silu_gate = (gate_fp32 * tl.sigmoid(gate_fp32)).to( + out_desc.dtype + ) + result = silu_gate * up + + out_desc.store([offs_m, offs_n + slice_id * slice_size], result) + else: + # Process full tile + gate = tlx.local_load(gate_tmem).to(out_desc.dtype) + up = tlx.local_load(up_tmem).to(out_desc.dtype) + + gate_fp32 = gate.to(tl.float32) + silu_gate = (gate_fp32 * tl.sigmoid(gate_fp32)).to(out_desc.dtype) + result = silu_gate * up + + out_desc.store([offs_m, offs_n], result) + + # Signal MMA that TMEM buffer is free + # pyre-ignore[6] + tlx.barrier_arrive(tmem_empty_bars[cur_tmem_buf], 1) + + processed_tiles += 1 + + # MMA Consumer: Computes both GEMMs: gate = X @ W_gate, up = X @ W_up + with tlx.async_task(num_warps=4, num_regs=232): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + k_tiles = tl.cdiv(K, BLOCK_K) + + processed_k_iters = 0 + processed_tiles = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid_swiglu( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + + cur_tmem_buf = processed_tiles % int(NUM_TMEM_BUFFERS) + tmem_write_phase = (processed_tiles // int(NUM_TMEM_BUFFERS)) & 1 + + # Wait for epilogue to finish + tlx.barrier_wait(tmem_empty_bars[cur_tmem_buf], tmem_write_phase ^ 1) + + # Perform K-dimension reduction for both GEMMs + for k in range(0, k_tiles): + buf = (processed_k_iters + k) % int(NUM_SMEM_BUFFERS) + + total_iters = processed_k_iters + k + dot_phase = (total_iters // int(NUM_SMEM_BUFFERS)) & 1 + + # Wait for x and w_gate to be loaded, then start GEMM1 + tlx.barrier_wait(smem_full_bars_x_gate[buf], dot_phase) + + # Transpose weight buffer for MMA + w_gate_trans = tlx.local_trans(w_gate_buffers[buf]) + + # GEMM 1: gate = X @ W_gate.T + tlx.async_dot( + x_buffers[buf], + w_gate_trans, + tmem_gate_buffers[cur_tmem_buf], + # pyre-ignore[6] + use_acc=(k > 0), + mBarriers=[smem_empty_bars[buf]], + out_dtype=tl.float32, + ) + + # Wait for w_up to be loaded before starting GEMM2 + tlx.barrier_wait(smem_full_bars_up[buf], dot_phase) + + w_up_trans = tlx.local_trans(w_up_buffers[buf]) + + # GEMM 2: up = X @ W_up.T + tlx.async_dot( + x_buffers[buf], + w_up_trans, + tmem_up_buffers[cur_tmem_buf], + # pyre-ignore[6] + use_acc=(k > 0), + mBarriers=[smem_empty_bars[buf]], + out_dtype=tl.float32, + ) + + # Wait for last MMA to complete + last_buf = (processed_k_iters + k_tiles - 1) % int(NUM_SMEM_BUFFERS) + last_total_iters = processed_k_iters + k_tiles - 1 + last_dot_phase = (last_total_iters // int(NUM_SMEM_BUFFERS)) & 1 + tlx.barrier_wait(smem_empty_bars[last_buf], last_dot_phase) + + # Signal epilogue that results are ready + # pyre-ignore[6] + tlx.barrier_arrive(tmem_full_bars[cur_tmem_buf], 1) + + processed_tiles += 1 + processed_k_iters += k_tiles + + # Producer: TMA loads for X, W_gate, W_up + with tlx.async_task(num_warps=1, num_regs=24): + start_pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + num_tiles = num_pid_m * num_pid_n + k_tiles = tl.cdiv(K, BLOCK_K) + + # Initialize phase tracking + processed_k_iters = 0 + + for tile_id in range(start_pid, num_tiles, NUM_SMS): + pid_m, pid_n = _compute_pid_swiglu( + tile_id, num_pid_in_group, num_pid_m, GROUP_M, NUM_SMS + ) + offs_m = pid_m * BLOCK_M + offs_n = pid_n * BLOCK_N + + for k in range(0, k_tiles): + buf = (processed_k_iters + k) % int(NUM_SMEM_BUFFERS) + + total_iters = processed_k_iters + k + load_phase = (total_iters // int(NUM_SMEM_BUFFERS)) & 1 + + # Wait for buffer to be free + tlx.barrier_wait(smem_empty_bars[buf], load_phase ^ 1) + + offs_k = k * BLOCK_K + + # Set expected bytes for x+w_gate barrier + tlx.barrier_expect_bytes( + smem_full_bars_x_gate[buf], + # pyre-ignore[6] + 2 * (BLOCK_M * BLOCK_K + BLOCK_K * BLOCK_N), + ) + + # Set expected bytes for w_up barrier + tlx.barrier_expect_bytes( + smem_full_bars_up[buf], + # pyre-ignore[6] + 2 * (BLOCK_K * BLOCK_N), + ) + + # Load x and w_gate first, signal smem_full_bars_x_gate + tlx.async_descriptor_load( + x_desc, + x_buffers[buf], + [offs_m, offs_k], + smem_full_bars_x_gate[buf], + ) + + # Weights are in [N, K] layout, load with [offs_n, offs_k] + tlx.async_descriptor_load( + w_gate_desc, + w_gate_buffers[buf], + [offs_n, offs_k], + smem_full_bars_x_gate[buf], + ) + + # Load w_up separately, signal smem_full_bars_up + tlx.async_descriptor_load( + w_up_desc, + w_up_buffers[buf], + [offs_n, offs_k], + smem_full_bars_up[buf], + ) + + processed_k_iters += k_tiles + + +@torch.fx.wrap +def triton_swiglu_fwd_tma_ws_persistent_tlx( + x: torch.Tensor, + w_gate: torch.Tensor, + w_up: torch.Tensor, +) -> torch.Tensor: + M, K = x.shape + N, K_gate = w_gate.shape + N_up, K_up = w_up.shape + + # Only bf16/fp16 supported by the kernel + supported_dtypes = (torch.bfloat16, torch.float16) + assert x.dtype in supported_dtypes, ( + f"x.dtype must be bfloat16 or float16, got {x.dtype}" + ) + assert w_gate.dtype in supported_dtypes, ( + f"w_gate.dtype must be bfloat16 or float16, got {w_gate.dtype}" + ) + assert w_up.dtype in supported_dtypes, ( + f"w_up.dtype must be bfloat16 or float16, got {w_up.dtype}" + ) + + assert K == K_gate, f"Incompatible dimensions: x.K={K}, w_gate.K={K_gate}" + assert K == K_up, f"Incompatible dimensions: x.K={K}, w_up.K={K_up}" + assert N == N_up, f"Incompatible dimensions: w_gate.N={N}, w_up.N={N_up}" + + # Allocate output + out = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return out + + M_BLOCK = triton.next_power_of_2(M) + + NUM_SMS = torch.cuda.get_device_properties("cuda").multi_processor_count + + # A dummy block value that will be overwritten by the hook + dummy_block = [1, 1] + + # pyre-ignore[6] + x_desc = TensorDescriptor(x, x.shape, x.stride(), dummy_block) + # pyre-ignore[6] + w_gate_desc = TensorDescriptor(w_gate, w_gate.shape, w_gate.stride(), dummy_block) + # pyre-ignore[6] + w_up_desc = TensorDescriptor(w_up, w_up.shape, w_up.stride(), dummy_block) + # pyre-ignore[6] + out_desc = TensorDescriptor(out, out.shape, out.stride(), dummy_block) + + def grid(meta): + BLOCK_M = meta["BLOCK_M"] + BLOCK_N = meta["BLOCK_N"] + return ( + min( + NUM_SMS, + triton.cdiv(M, BLOCK_M) * triton.cdiv(N, BLOCK_N), + ), + ) + + _swiglu_fwd_tma_ws_persistent[grid]( + x_desc, + w_gate_desc, + w_up_desc, + out_desc, + M, + N, + K, + M_BLOCK, + NUM_SMS=NUM_SMS, + NUM_SMEM_BUFFERS=4, + NUM_TMEM_BUFFERS=2, + EPILOGUE_SUBTILE=2, + ) + return out + + +# ============================================================================= +# Standard fused SwiGLU kernel for A100/H100 (non-TLX path). +# +# Fuses silu(x @ W_gate^T) * (x @ W_up^T) into a single kernel launch. +# Uses standard Triton pointer arithmetic (no TMA), works on SM80+. +# +# Key optimization: x is loaded from HBM ONCE and reused for both GEMMs. +# Activation (silu * up) is computed in float32 registers, no HBM round-trip. +# +# Weight layout: expects [N, K] (nn.Linear native format). +# The wrapper transposes to [K, N] for the GEMM internally. +# ============================================================================= + + +def _get_swiglu_fwd_configs() -> List[triton.Config]: + """ + Autotune configs for the standard (non-TLX) fused SwiGLU kernel. + + Two float32 accumulators (gate + up) double register pressure vs single + GEMM, so smaller block sizes are included. + """ + configs = [ + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "BLOCK_K": 64, "GROUP_M": 8}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=3, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=3, + num_warps=8, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 128, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=3, + num_warps=8, + ), + triton.Config( + {"BLOCK_M": 32, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 32, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=4, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "BLOCK_K": 64, "GROUP_M": 8}, + num_stages=3, + num_warps=8, + ), + ] + if torch.version.hip: + hip_num_stages = 2 if triton.__version__ >= "3.2.0" else 0 + configs.extend( + [ + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 128, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=hip_num_stages, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 128, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=hip_num_stages, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 128, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=hip_num_stages, + num_warps=4, + ), + triton.Config( + {"BLOCK_M": 64, "BLOCK_N": 64, "BLOCK_K": 32, "GROUP_M": 8}, + num_stages=hip_num_stages, + num_warps=4, + ), + ] + ) + return configs + + +@triton_autotune( + configs=_get_swiglu_fwd_configs(), + key=["M_BLOCK", "N", "K"], +) +@triton.jit +def _swiglu_fwd_kernel( + # Pointers to input/output tensors + x_ptr, # [M, K] input activation + w_gate_ptr, # [K, N] gate weight (already transposed from [N, K]) + w_up_ptr, # [K, N] up weight (already transposed from [N, K]) + out_ptr, # [M, N] output = silu(x @ w_gate) * (x @ w_up) + # Matrix dimensions + M, # rows in x (batch_size * seq_len) + N, # output dimension (hidden_dim) + K, # input/reduction dimension (input_dim) + M_BLOCK, # next_power_of_2(M) for stable autotuning + # Strides + stride_xm, + stride_xk, + stride_wgk, + stride_wgn, + stride_wuk, + stride_wun, + stride_om, + stride_on, + # Compile-time constants + BLOCK_M: tl.constexpr, + BLOCK_N: tl.constexpr, + BLOCK_K: tl.constexpr, + GROUP_M: tl.constexpr, + ALLOW_TF32: tl.constexpr, +): + """ + Fused SwiGLU forward: out = silu(x @ W_gate) * (x @ W_up). + + Each thread block computes one [BLOCK_M, BLOCK_N] output tile. + Two accumulators share the same x tile loads (the fusion benefit). + """ + # -- Step 1: Compute tile coordinates with grouped ordering (L2 reuse) -- + pid = tl.program_id(axis=0) + num_pid_m = tl.cdiv(M, BLOCK_M) + num_pid_n = tl.cdiv(N, BLOCK_N) + num_pid_in_group = GROUP_M * num_pid_n + group_id = pid // num_pid_in_group + first_pid_m = group_id * GROUP_M + + group_size_m = min(num_pid_m - first_pid_m, GROUP_M) + pid_in_group = pid % num_pid_in_group + pid_m = first_pid_m + (pid_in_group % group_size_m) + pid_n = pid_in_group // group_size_m + + # -- Step 2: Set up pointers for x, w_gate, w_up tiles -- + offs_m = tl.arange(0, BLOCK_M) + offs_n = tl.arange(0, BLOCK_N) + offs_k = tl.arange(0, BLOCK_K) + mask_m = (pid_m * BLOCK_M + offs_m)[:, None] < M + mask_n = (pid_n * BLOCK_N + offs_n)[None, :] < N + # [BLOCK_M, BLOCK_K] + x_ptrs = ( + x_ptr + + (pid_m.to(tl.int64) * BLOCK_M + offs_m)[:, None] * stride_xm + + offs_k[None, :] * stride_xk + ) + # [BLOCK_K, BLOCK_N] + wg_ptrs = ( + w_gate_ptr + + offs_k[:, None] * stride_wgk + + (pid_n.to(tl.int64) * BLOCK_N + offs_n)[None, :] * stride_wgn + ) + + # [BLOCK_K, BLOCK_N] + wu_ptrs = ( + w_up_ptr + + offs_k[:, None] * stride_wuk + + (pid_n.to(tl.int64) * BLOCK_N + offs_n)[None, :] * stride_wun + ) + + # -- Step 3: K-loop - two GEMMs sharing the same x tile -- + acc_gate = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + acc_up = tl.zeros((BLOCK_M, BLOCK_N), dtype=tl.float32) + + for k in range(0, tl.cdiv(K, BLOCK_K)): + mask_k = offs_k[None, :] < K - k * BLOCK_K + x = tl.load(x_ptrs, mask=mask_m & mask_k, other=0.0) + mask_k = offs_k[:, None] < K - k * BLOCK_K + wg = tl.load(wg_ptrs, mask=mask_k & mask_n, other=0.0) + wu = tl.load(wu_ptrs, mask=mask_k & mask_n, other=0.0) + + acc_gate += tl.dot(x, wg, allow_tf32=ALLOW_TF32) + acc_up += tl.dot(x, wu, allow_tf32=ALLOW_TF32) + + x_ptrs += BLOCK_K * stride_xk + wg_ptrs += BLOCK_K * stride_wgk + wu_ptrs += BLOCK_K * stride_wuk + + # -- Step 4: Apply SwiGLU activation in registers (no HBM round-trip) -- + gate_activated = acc_gate * tl.sigmoid(acc_gate) # silu + result = (gate_activated * acc_up).to(out_ptr.dtype.element_ty) + + # -- Step 5: Store result -- + offs_m = pid_m * BLOCK_M + offs_m + offs_n = pid_n * BLOCK_N + offs_n + out_ptrs = out_ptr + offs_m[:, None] * stride_om + offs_n[None, :] * stride_on + tl.store(out_ptrs, result, mask=mask_m & mask_n) + + +def triton_swiglu_fwd( + x: torch.Tensor, + w_gate: torch.Tensor, + w_up: torch.Tensor, +) -> torch.Tensor: + """ + Forward pass of fused SwiGLU (non-TLX path). Works on A100/H100/MI300X. + + Computes: silu(x @ w_gate^T) * (x @ w_up^T) + + Args: + x: [M, K] input tensor + w_gate: [N, K] gate weight (nn.Linear format) + w_up: [N, K] up weight (nn.Linear format) + + Returns: + [M, N] output tensor + """ + M, K = x.shape + N, K_gate = w_gate.shape + N_up, K_up = w_up.shape + assert K == K_gate, f"x.K={K} != w_gate.K={K_gate}" + assert K == K_up, f"x.K={K} != w_up.K={K_up}" + assert N == N_up, f"w_gate.N={N} != w_up.N={N_up}" + + out = torch.empty((M, N), device=x.device, dtype=x.dtype) + if M == 0 or N == 0: + return out + + M_BLOCK = triton.next_power_of_2(M) + + # Transpose weights from [N, K] to [K, N] for the GEMM kernel + w_gate_t = w_gate.t().contiguous() + w_up_t = w_up.t().contiguous() + + grid = lambda meta: ( # noqa E731 + triton.cdiv(M, meta["BLOCK_M"]) * triton.cdiv(N, meta["BLOCK_N"]), + ) + + _swiglu_fwd_kernel[grid]( + x, + w_gate_t, + w_up_t, + out, + M, + N, + K, + M_BLOCK, + x.stride(0), + x.stride(1), + w_gate_t.stride(0), + w_gate_t.stride(1), + w_up_t.stride(0), + w_up_t.stride(1), + out.stride(0), + out.stride(1), + ALLOW_TF32=torch.backends.cuda.matmul.allow_tf32, + ) + return out + + +def triton_swiglu( + x: torch.Tensor, + w_gate: torch.Tensor, + w_up: torch.Tensor, +) -> torch.Tensor: + if is_sm100_plus() and TMA_AVAILABLE and HAS_TLX: + # Blackwell: use the fast TLX persistent kernel with TMA + _, K = x.shape + N, _ = w_gate.shape + assert K % 16 == 0 and N % 16 == 0, ( + f"K ({K}) and N ({N}) must be divisible by 16 for TMA alignment" + ) + return triton_swiglu_fwd_tma_ws_persistent_tlx(x, w_gate, w_up) + else: + # A100/H100: use the standard fused kernel + return triton_swiglu_fwd(x, w_gate, w_up) diff --git a/recommendation_v4/generative_recommenders/ops/utils.py b/recommendation_v4/generative_recommenders/ops/utils.py new file mode 100644 index 000000000..16edd99a9 --- /dev/null +++ b/recommendation_v4/generative_recommenders/ops/utils.py @@ -0,0 +1,124 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-ignore-all-errors + +import functools +import os + +import torch + + +class _PlainFuncWrapper: + """Thin wrapper around a plain function that provides no-op register_fake + and register_kernel methods, mirroring the CustomOpDef API so that + downstream @func.register_fake / func.register_kernel("cpu") calls + don't break when the function is not wrapped as a custom op.""" + + def __init__(self, func): + self._func = func + functools.update_wrapper(self, func) + + def __call__(self, *args, **kwargs): + return self._func(*args, **kwargs) + + def register_fake(self, fake_func): + return fake_func + + def register_kernel(self, device): + def inner(func): + return func + + return inner + + +def maybe_register_custom_op(op_name, mutates_args): + """ + Conditionally registers a function as a torch custom op. + + When AOTI_LOWER is set in the environment, the function is returned + unwrapped so that torch.export / Dynamo can trace through the plain + Python implementation instead of treating the custom op as opaque. + """ + + def decorator(func): + if os.environ.get("AOTI_LOWER"): + return _PlainFuncWrapper(func) + return torch.library.custom_op(op_name, func, mutates_args=mutates_args) + + return decorator + + +def is_sm100_plus() -> bool: + """ + Check if this is a Blackwell Datacenter GPU. + These are between 100 and 103 for B200-GB300. + """ + if not torch.cuda.is_available(): + return False + props = torch.cuda.get_device_properties(0) + return props.major == 10 and (props.minor >= 0 and props.minor <= 3) + + +def is_sm90() -> bool: + if not torch.cuda.is_available(): + return False + props = torch.cuda.get_device_properties(0) + return props.major == 9 and props.minor == 0 + + +def is_sm90_plus() -> bool: + return is_sm100_plus() or is_sm90() + + +@functools.lru_cache(maxsize=None) +def is_amd_mi350() -> bool: + """Detect an AMD Instinct MI350-class GPU (gfx950) running under ROCm. + + MI350 benefits from the same multi-row, separated-RNG layer-norm-mul-dropout + path as Blackwell datacenter parts (sm_100), so it is gated together with + is_sm100_plus() at the kernel dispatch sites. + """ + if not torch.cuda.is_available(): + return False + if getattr(torch.version, "hip", None) is None: + return False + try: + arch = torch.cuda.get_device_properties(0).gcnArchName or "" + except (AssertionError, RuntimeError, AttributeError): + return False + return "gfx950" in arch + + +def use_separated_rng_ln_mul_dropout() -> bool: + """Hardware that should use the autotuned, multi-row ``_ln_mul_dropout_fwd_rng`` + kernel with a precomputed dropout mask instead of the legacy single-row, + fused-RNG ``_ln_mul_dropout_fwd`` kernel. + + Blackwell datacenter GPUs (sm_100-103) and AMD MI350 (gfx950) both prefer the + separated-RNG path: it batches rows per program and lets the backward reuse the + same mask, which is a large win over launching one program per row. + """ + return is_sm100_plus() or is_amd_mi350() + + +def copy_if_different_ptr(dst: torch.Tensor, src: torch.Tensor) -> None: + if torch.compiler.is_compiling(): + # .data_ptr() will break PT2 + dst.copy_(src) + else: + if dst.data_ptr() != src.data_ptr(): + dst.copy_(src) diff --git a/recommendation_v4/generative_recommenders/tests/test_common.py b/recommendation_v4/generative_recommenders/tests/test_common.py new file mode 100644 index 000000000..be3823d67 --- /dev/null +++ b/recommendation_v4/generative_recommenders/tests/test_common.py @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/env python3 + +# pyre-strict + +import unittest + +import torch +from generative_recommenders.common import switch_to_contiguous_if_needed + + +class SwitchToContiguousIfNeededTest(unittest.TestCase): + def test_torchscript_does_not_compile_fx_tracing_helper(self) -> None: + class ContiguousModule(torch.nn.Module): + def forward(self, x: torch.Tensor) -> torch.Tensor: + return switch_to_contiguous_if_needed(x) + + scripted = torch.jit.script(ContiguousModule()) + x = torch.arange(12).reshape(3, 4).transpose(0, 1) + + out = scripted(x) + + self.assertTrue(torch.equal(out, x)) + self.assertTrue(out.is_contiguous()) + + +if __name__ == "__main__": + unittest.main() diff --git a/recommendation_v4/md5sums_yambda_5b_processed.txt b/recommendation_v4/md5sums_yambda_5b_processed.txt new file mode 100644 index 000000000..82998cca3 --- /dev/null +++ b/recommendation_v4/md5sums_yambda_5b_processed.txt @@ -0,0 +1,22 @@ +# MD5 checksums for the preprocessed yambda-5b dataset (processed_5b/). +# +# Format: standard `md5sum` output -> " ". +# Paths are relative to ${DLRM_DATA_PATH}/processed_5b/. +# +# These hashes are PLACEHOLDERS (TODO). They must be generated from a canonical +# preprocessing run before this benchmark is submitted, e.g.: +# +# cd "${DLRM_DATA_PATH}/processed_5b" +# md5sum train_sessions.parquet test_events.parquet session_index.parquet \ +# item_popularity.npy split_meta.json \ +# > /md5sums_yambda_5b_processed.txt +# +# Until then `verify_dataset.sh` falls back to an existence/layout check and +# warns that checksums are not yet pinned. +# +# TODO(rcp/data): replace the lines below with real md5 hashes. +TODO_GENERATE_HASH train_sessions.parquet +TODO_GENERATE_HASH test_events.parquet +TODO_GENERATE_HASH session_index.parquet +TODO_GENERATE_HASH item_popularity.npy +TODO_GENERATE_HASH split_meta.json diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed130327877.log b/recommendation_v4/rcp_logs/gbs_16384/seed130327877.log new file mode 100644 index 000000000..110652e87 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed130327877.log @@ -0,0 +1,679 @@ +:::MLLOG {"namespace": "", "time_ms": 1784929469520, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784929469520, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784929490527, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784929490528, "event_type": "POINT_IN_TIME", "key": "seed", "value": 130327877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784929490529, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784929490529, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784929490529, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784929490529, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784929490532, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784929490533, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784930372702, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784930372703, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784930373031, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784930497305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741058111190796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930510850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781070709228516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930524438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829509913921356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930538163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851675391197205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930551938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880226016044617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930565816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387527883052826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784930579673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852539658546448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930579674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930579674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930659350, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014888048171997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930659351, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930659351, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784930673060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760338723659515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930686909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772650063037872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930700676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381089687347412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930714538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137943297624588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930728544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379503756761551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930742563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722749054431915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930756866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821694254875183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930756867, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930756867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930800573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016469955444336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930800574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930800574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784930814480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787244260311127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930828603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738864660263062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930842411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742533326148987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930856829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735651969909668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930870497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757987320423126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930884839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771796226501465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930898673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721787929534912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930898673, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930898674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930942341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018129348754883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930942342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930942342, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784930956242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819126784801483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930970563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744735717773438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930984623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632023334503174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784930998844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377808004617691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931013303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369735300540924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931027294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671943545341492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931041715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664554059505463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931041716, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931041716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931084394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020522475242615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931084395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931084395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784931098182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360839307308197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931112228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372012495994568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931126395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694654405117035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931140098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646364212036133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931153648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346900761127472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931167548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641348481178284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931181729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350817084312439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931181730, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931181730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931225260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023185014724731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931225260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931225260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784931239168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362655907869339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931253061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568076491355896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931267284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714197278022766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931281418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13608047366142273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931295057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340285837650299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931308649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476930558681488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931322494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731053471565247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931322494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931322495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931365182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026360750198364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931365183, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931365183, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784931379536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364598423242569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931393838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505887985229492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931407978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520953059196472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931422095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341310441493988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931436448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438838720321655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931451096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387593626976013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931465211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440167903900146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931465212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931465212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931507066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030504465103149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931507067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931507067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784931521396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355876505374908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931535869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361204981803894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931550692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822153210639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931565313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343259692192078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931580004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582974672317505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931594402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454318046569824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931609227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653114438056946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931609228, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931609228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931651009, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036609768867493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931651009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931651009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931665704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650506734848022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931680340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366949826478958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931695045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467320799827576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931709681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969735264778137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931724383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221287727355957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931738830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352198898792267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931753143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485737144947052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931753189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931753190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931794974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5044026374816895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931794974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931794975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784931809471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349785029888153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931823936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877029180526733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931839038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098996877670288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931853236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368754506111145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931867403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216570019721985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931881371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128334283828735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931895951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325334906578064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931895951, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931895952, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931939435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052188038825989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931939435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931939436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784931953333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316993087530136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931968311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341438740491867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931982440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777823209762573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931996686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291271150112152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932010878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392679393291473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932025585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824705243110657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932039873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297684907913208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932039874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932039874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932085320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5063700079917908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932085320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932085321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784932100147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472974300384521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932114219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330287605524063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932128489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441359996795654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932142686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486254215240479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932157419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614408671855927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932171760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350197672843933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932186348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678127527236938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932186348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932186349, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932229223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077407956123352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932229223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932229224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784932243925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342841386795044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932258693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305330991744995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932273444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387757539749146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932288479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14049670100212097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932303322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033631443977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932318268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240987062454224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932333119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821211457252502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932333120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932333120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932376636, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090627670288086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932376637, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932376637, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784932391750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12775777280330658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932406502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108964264392853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932421443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370779573917389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932436284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532330095767975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932451554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321917027235031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932465908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12864068150520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932480436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127950519323349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932480437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932480437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932522820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5104547739028931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932522821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932522821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784932537198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011324405670166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932551620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134669691324234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932566335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14207105338573456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932580797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340215504169464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932595808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314902901649475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932610095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336403489112854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932624666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590475916862488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932624666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932624667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932668645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5120578408241272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932668645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932668645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784932682826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13468238711357117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932697281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632658123970032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932712015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551592826843262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932726831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335202634334564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932741597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343533843755722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932756417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292402029037476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932770947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249844312667847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932770948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932770948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932813744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5137181878089905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932813746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932813746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784932828301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192260265350342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932842986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430948555469513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932857932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14167660474777222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932873136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338476538658142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932888159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342276930809021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932903079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323634386062622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932917875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316604495048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932917876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932917876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932966605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5156402587890625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932966606, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932966606, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784932981255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137608140707016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932996085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301944732666016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933010750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12875109910964966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933025604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005080819129944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933040561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380065381526947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933055684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316547393798828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933070551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621389865875244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933070552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933070552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933113600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5178365111351013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933113601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933113601, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784933128523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367304027080536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933143122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641595840454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933157832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313653290271759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933172457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990793585777283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933187291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314408779144287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933202465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522374629974365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933217745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259953260421753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933217746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933217746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933262076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.520391047000885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933262077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933262077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784933276857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598811626434326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933292191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13226543366909027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933307462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378733366727829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933322696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136343315243721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933338014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385926604270935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933353037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428834080696106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933368558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13920357823371887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933368559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933368559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933411612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5229726433753967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933411612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933411613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784933427081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508540391921997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933442453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14768365025520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933458020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350589394569397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933473178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308514654636383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933488527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433343172073364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933503656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133161261677742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933518895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662147521972656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933518895, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933518896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933562185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5260679125785828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933562185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933562186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784933577996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611501455307007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933593421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530835509300232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933608847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374775648117065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933623690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365629315376282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933638563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331365406513214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933653296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12933799624443054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933667955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560515105724335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933667956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933667956, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933710975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5298409461975098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933710976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933710976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784933725678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263115286827087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933740474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244135677814484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933755361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496941328048706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933770072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353374421596527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933784942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662564754486084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933799960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12875956296920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933814817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298283338546753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933814818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933814818, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933859188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5354326367378235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933859189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933859190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784933874057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327492594718933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933888909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12703540921211243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933903565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133143350481987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933918242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322110891342163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933933405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697896361351013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933948660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323021501302719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933963292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415291905403137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933963292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784933963293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784934009049, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5434387922286987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784934009050, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784934009050, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784934024175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294630765914917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934038828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031145930290222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934053341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304529309272766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934067982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977255880832672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934082320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12775647640228271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934096789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12317872792482376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934111688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127197265625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934111689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934111689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934155480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.554966151714325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934155481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934155481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784934170330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577383756637573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934185310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332085132598877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934199779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895773887634277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934215143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328003883361816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934230068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763072550296783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934244972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295953392982483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934260189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268189549446106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934260190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934260190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934304744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5694131255149841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934304745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934304745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784934319562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884759902954102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934334413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706129252910614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934349651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322837769985199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934364911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229209184646606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934380264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118326663970947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934395225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331058621406555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934410435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302037537097931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934410436, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934410437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934453484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5860764384269714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934453485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934453485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784934468442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292733997106552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934483266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129511296749115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934498144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903490662574768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934513034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209083676338196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934527714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13267509639263153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934542208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289127916097641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934557389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297374665737152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934557390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934557390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934600838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.599346935749054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934600838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934600839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784934615786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751734256744385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934631061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995287775993347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934645518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722760438919067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934661079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265142858028412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934676600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613072991371155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934692001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300499439239502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934707638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094021379947662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934707639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934707639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934750436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6118835806846619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934750437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934750437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784934765432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840168178081512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934780760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12602829933166504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934796459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288432478904724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934812216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079264760017395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934827842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317555457353592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934843034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721608579158783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934858905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12741565704345703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934858905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934858906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934902955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6247787475585938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934902956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934902956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784934918060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12746237218379974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934933136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1251930296421051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934948149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12620909512043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934962954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330440640449524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934978098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287929266691208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934993453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12359479069709778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935008973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12615445256233215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935008973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935008973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935054359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6358637809753418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935054360, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935054360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784935069274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12257000803947449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935085173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12526625394821167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935100360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12452530860900879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935115392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235341727733612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935130302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12219133973121643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935145389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11879190057516098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935160405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12145138531923294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935160406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935160406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935205388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6496809124946594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935205389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935205389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784935220798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12360332161188126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935235989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201949417591095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935251176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310137808322906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935266374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11930988729000092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935281504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460000276565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935296561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515715718269348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935311457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312208324670792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935311457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935311457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935354747, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6717750430107117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935354748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935354748, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784935369753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890266835689545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935384511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146303340792656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935399085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977821052074432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935413853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11766226589679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935429080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679330468177795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935444572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520588397979736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935459584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744430124759674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935459585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935459585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935504495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7100310325622559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935504496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935504496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784935519364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425633728504181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935535232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545336991548538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935550317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11790451407432556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935565221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271555721759796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935580105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145804822444916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935594916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220148205757141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935609864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820285975933075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935609907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935609908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935653402, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7336806058883667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935653403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935653403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784935668188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309146881103516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935683610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122581362724304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935698758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780869960784912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935714294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128823459148407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935729189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230617761611938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935744716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813097447156906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935759774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595645010471344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935759774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935759774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935802712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7497020363807678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935802713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935802713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784935818036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459602415561676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935833294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259129643440247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935848655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104150265455246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935863656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031477153301239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935878626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394148528575897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935893274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772160232067108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935908559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379587650299072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935908559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935908559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935951588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7562284469604492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935951589, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935951590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784935966687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685779690742493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935982175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278510838747025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935997358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565368622541428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936013049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143715143203735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936028272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478415876626968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936043372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841018706560135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936058478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750117897987366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936058479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936058479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936102249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7589854598045349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936102250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936102250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784936117451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947012156248093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936132604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869793593883514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936147693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435183137655258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936162985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937294363975525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936178490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301885545253754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936193962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363711416721344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936209587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160542070865631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936209588, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936209588, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936253034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7611272931098938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936253035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936253036, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784936268387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816161334514618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936283427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957103967666626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936299681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11981431394815445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936315417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12345238029956818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936330862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11666906625032425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936346488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183468997478485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936361979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626444011926651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936361980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936361980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936404840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634645700454712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936404841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936404841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784936420269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558946222066879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936435827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910797864198685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936451608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11857803165912628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936467236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146941035985947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936483286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491192132234573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936498649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437024176120758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936514729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535179078578949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936514730, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936514730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936558442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649858593940735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936558443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936558443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784936574085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188303530216217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936589745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751298815011978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936605274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490494221448898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936620741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476822197437286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936636291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11909498274326324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936651957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816273093223572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936667981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483326554298401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936667982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936667982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936712152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7650060057640076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936712152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936712152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784936727885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671522259712219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936743775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888705402612686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936758838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898895561695099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936774301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11372948437929153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936789626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791391134262085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936804688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041310653090477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936819664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755736172199249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936819665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936819665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936863916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651113271713257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936863917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936863917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784936879111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038492619991302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936894208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148344874382019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936909157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832541435956955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936924043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596492886543274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936939512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313886404037476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936954865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09733660519123077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936970406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737061500549316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936970407, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784936970407, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784937014491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656068801879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784937014492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784937014493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784937029634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593140870332718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937044351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028829216957092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937060023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956215649843216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937075214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090565174818039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937090058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705958306789398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937104744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178963840007782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937119763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818703472614288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937119764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937119764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937163033, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766601026058197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937163034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937163034, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784937178204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594727098941803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937192936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602781176567078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937207946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059771478176117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937223188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792025923728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937238533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076001524925232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937253796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279842257499695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937269423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060839146375656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937269424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937269424, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937313566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676143646240234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937313567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937313567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784937328399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563057661056519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937343740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379929095506668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937359213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115039885044098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937374679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953591763973236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937390018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053420677781105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937405510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165087878704071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937420945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525807738304138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937420945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937420945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937463902, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685555815696716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937463902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937463902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784937479099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050298810005188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937494528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096332430839539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937509549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055753231048584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937525075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108141839504242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937540586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684279352426529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937556260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361512005329132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937571881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075699329376221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937571882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937571882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937616050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685883045196533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937616051, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937616051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784937630836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007748693227768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937646377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475396364927292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937661444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770799219608307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937676752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446397215127945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937691911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302675515413284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937707044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789551585912704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937722425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043740063905716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937722426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937722426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937765773, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698070406913757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937765774, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937765774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784937780838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073603481054306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937795814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043315902352333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937810550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095409169793129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937825774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697400569915771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937841066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123005300760269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937856384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818073153495789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937871570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174861341714859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937871571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937871571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937915439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695820331573486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937915440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937915441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784937930224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668941587209702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937945748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022358238697052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937961071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548575222492218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937976468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243897140026093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937991683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087808907032013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938006864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134478896856308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938022251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984211415052414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938022251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938022252, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938065455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770494818687439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938065456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938065456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784938080897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604013502597809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938096225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11168843507766724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938111532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087055504322052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938127095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099413245916367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938142613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324885487556458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938158160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370700597763062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938174077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495191067457199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938174077, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938174078, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938217140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704154849052429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938217141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938217141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784938232437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889706015586853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938247197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062984377145767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938262925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757730692625046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938278521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731936037540436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938293863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553119122982025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938309228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564127564430237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938324697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767528414726257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938324697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938324698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938368681, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713645696640015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938368681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938368681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784938384239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193837225437164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938399410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445696651935577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938414814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084121465682983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938430505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152133345603943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938446115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154436320066452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938461714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301317811012268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938477646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415009200572968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938477647, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938477647, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938522193, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711153030395508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938522194, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938522194, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784938537326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031399667263031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938552965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778987407684326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938568770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795535147190094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938584733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152028888463974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938600217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186620444059372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938615715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073325574398041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938631599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614029109477997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938631600, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784938631600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784938676025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716622948646545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784938676026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784938676026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed149983452.log b/recommendation_v4/rcp_logs/gbs_16384/seed149983452.log new file mode 100644 index 000000000..f90bed795 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed149983452.log @@ -0,0 +1,820 @@ +:::MLLOG {"namespace": "", "time_ms": 1784867221561, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784867221562, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784867241789, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "seed", "value": 149983452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784867241799, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784867241812, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784867241812, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784868083329, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784868083330, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784868083648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784868260319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772815465927124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868273993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815942406654358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868287657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840842247009277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868301553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386566162109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868315530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387680172920227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868329366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854938745498657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868343125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837754726409912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868343125, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868343126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868388379, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001096725463867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868388380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868388380, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868402017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381252110004425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868415677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381990760564804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868429607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382634937763214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868443628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821136951446533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868457397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382584571838379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868471683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782069087028503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868486160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727018237113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868486161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868486161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868529410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002773404121399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868529411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868529411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868543340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810822367668152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868557589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767564296722412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868571490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777995109558105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868585824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760356605052948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868599498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379493921995163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868613837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380046308040619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868627967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732922077178955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868627967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868627968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868670621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004666447639465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868670621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868670622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868684629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829371333122253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868699219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753271102905273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868713259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368006467819214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868727463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794606924057007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868742323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732090592384338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868756414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691341876983643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868770881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711164891719818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868770882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868770882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868812764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007104277610779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868812765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868812765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868826379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634498417377472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868840238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738033175468445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868854551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714037835597992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868868395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366560161113739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868882159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351199448108673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868896121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666412234306335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868910218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538694381713867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868910219, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868910219, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868953291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010009407997131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868953292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868953292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868966990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362731158733368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868980753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598227500915527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868994947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727161288261414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869009156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136247456073761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869022952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448652625083923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869036706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500705361366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869050655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745510578155518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869050655, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869050656, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869094686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013360381126404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869094687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869094687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869108984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668543100357056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869123165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352468729019165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869136780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553249835968018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869150689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454163074493408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869164788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462460041046143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869179469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426440954208374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869193884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465335965156555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869193884, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869193884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869236578, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017980337142944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869236578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869236579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869251147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336870938539505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869265704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363849937915802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869280592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813598453998566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869294751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360777497291565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869309198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585391640663147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869323367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346030831336975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869338114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655516505241394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869338115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869338115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869382642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024309754371643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869382642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869382642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869397331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633349537849426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869412000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366102695465088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869426577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467179238796234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869441081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979018688201904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869455536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233448565006256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869469609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334487497806549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869483523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486891984939575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869483577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869483578, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869526963, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031209588050842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869526963, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869526963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869541146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501936197280884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869555274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894396483898163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869570133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309923678636551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869584341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687478005886078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869598727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322268843650818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869612869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117389380931854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869627518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324489414691925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869627518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869627519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869670861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038923621177673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869670861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869670862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869684831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316559910774231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869699636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410931825637817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869713575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12776678800582886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869727600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948453426361084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869741894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337880790233612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869756495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12821874022483826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869770527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297735869884491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869770528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869770528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869814656, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049933791160583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869814656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869814657, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869829852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469770550727844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869843805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330922544002533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869858192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447514176368713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869872523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492733240127563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869887325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625238835811615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869901686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364478945732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869916268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695815205574036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869916269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869916269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869960066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064151287078857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869960066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869960066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869974731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432876765727997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869989404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305205523967743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870004077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338917464017868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870019011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14038683474063873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870033872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039377331733704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870048858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235123455524445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870063561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820785284042358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870063562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870063562, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870105457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507748007774353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870105458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870105458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870120485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402431666851044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870135269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311700940132141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870150277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704606890678406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870165132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354977786540985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870180268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239121437072754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870194643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879443168640137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870209043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12803059816360474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870209044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870209044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870251309, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5091164708137512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870251309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870251309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870265820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301184892654419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870280297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347193717956543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870295023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14224904775619507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870309432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430684804916382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870324231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131514310836792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870338428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373112678527832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870352798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598880171775818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870352799, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870352799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870396007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107304453849792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870396007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870396007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870410227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470184803009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870424440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639642298221588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870438987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135719895362854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870453573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331528276205063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870468195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431167602539062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870482971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294729590415955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870497412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246354460716248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870497413, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870497414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870540706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5124543905258179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870540707, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870540707, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870555068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177615404129028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870569490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344207525253296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870584122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1419132947921753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870598954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436853885650635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870613620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13427215814590454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870628277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228777050971985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870642589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312450051307678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870642590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870642590, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870686825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5145044922828674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870686825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870686825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784870700829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844549655914307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870715065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300520181655884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870729105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877511978149414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870743261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002096116542816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870757692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793738186359406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870772318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159775733947754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870786967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604898750782013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870786967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870786968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870832213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5168299674987793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870832213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870832213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870846693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672715425491333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870860945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363033652305603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870875481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13126802444458008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870889729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996044754981995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870904345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314932107925415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870919169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532155752182007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870934042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326977014541626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870934042, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870934043, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870977418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5195149779319763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870977418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870977418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870991867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604511320590973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871006909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234862685203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871022010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791608810424805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871036659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639384508132935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871051645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855662941932678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871066427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435132801532745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871081607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13917629420757294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871081608, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871081608, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871125590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5223249197006226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871125590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871125590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871140684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494080305099487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871155757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14973768591880798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871171080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510572910308838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871186117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330367624759674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871201469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428595662117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871216568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315477967262268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871231839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365164965391159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871231839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871231840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871274811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5255382061004639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871274811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871274811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871289904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624602556228638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871304958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351504623889923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871320095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375455141067505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871334689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336391270160675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871349237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332518756389618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871363787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945963442325592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871378203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12555891275405884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871378204, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871378204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871421008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5293525457382202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871421009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871421009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871435507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273680210113525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871450245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252326846122742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871464960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511598110198975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871479579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354249268770218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871494226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672424852848053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871508945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12888166308403015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871523615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12985894083976746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871523616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871523616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871566554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5349465012550354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871566555, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871566555, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871581260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249850273132324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871596079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706145644187927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871610897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315142691135406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871625909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203096389770508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871641155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697166204452515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871656578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216593861579895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871671331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422314822673798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871671331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871671332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871713717, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.54236900806427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871713717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871713717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784871728556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294289380311966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871743064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034021854400635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871757345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13068878650665283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871771941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990888953208923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871786168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770181894302368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871800663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12336447089910507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871815446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714539468288422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871815447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871815447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871857465, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5525143146514893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871857465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871857466, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784871872230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573843240737915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871887054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331949710845947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871901289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391291618347168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871916548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364344835281372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871931616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12785980105400085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871946631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964238226413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871961841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713074684143066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871961842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784871961842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872004367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5651109218597412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872004368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872004368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872019272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289408802986145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872034294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751975655555725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872049712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325790286064148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872065252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282522559165955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872080799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142193853855133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872095819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382871448993683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872111131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053035736083984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872111132, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872111133, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872153383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5805420875549316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872153384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872153384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872168178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297227293252945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872182941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986285984516144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872197712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978605926036835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872212501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132462739944458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872226909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368993997573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872241189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948079407215118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872256247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042142987251282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872256248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872256248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872298899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5944309234619141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872298900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872298900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872313707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284046769142151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872328869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082963228225708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872343322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12869814038276672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872358955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12781338393688202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872374529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751121819019318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872390083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185951113700867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872405750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254693150520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872405750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872405751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872448312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6092762351036072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872448312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872448312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872463484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13016945123672485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872478963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808242440223694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872494856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304662823677063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872510695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209816813468933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872526452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339239358901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872541714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12923569977283478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872557392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12887969613075256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872557393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872557393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872600796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6244020462036133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872600796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872600797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872615726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976551055908203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872630833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12823563814163208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872646041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12930944561958313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872660927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536153733730316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872675945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312510073184967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872691260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12624222040176392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872706785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984615564346313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872706786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872706786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872751177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6381945013999939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872751178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872751178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784872766215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12668848037719727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872781800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811246514320374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872797025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12778359651565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872812090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274714171886444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872827029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256883144378662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872842023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12291957437992096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872856922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12433032691478729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872856923, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872856923, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872899371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6545944213867188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872899372, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872899372, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784872915036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722235918045044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872930523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354118376970291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872945655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172906756401062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872961014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324897199869156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872976045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887331306934357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872990884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12011583894491196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873005833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772780865430832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873005834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873005834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873048946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6724461317062378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873048946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873048947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873063942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227400004863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873078557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11845839023590088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873093196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540408432483673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873107975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12178345024585724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873123177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978889256715775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873138447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919917166233063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873153419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026335507631302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873153420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873153420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873197402, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7056285738945007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873197403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873197403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873211999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685685813426971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873227413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701890826225281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873242529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12067495286464691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873257302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152673214673996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873272231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358959227800369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873286912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546924710273743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873301897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077293753623962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873301949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873301949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873345179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7284626960754395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873345180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873345180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873359881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633269488811493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873375208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139206513762474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873390263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834435909986496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873405781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373282223939896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873420569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334877461194992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873436037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12116408348083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873450954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846187174320221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873450955, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873450955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873494357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7436291575431824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873494358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873494358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873509326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665308475494385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873524307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491361260414124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873539488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215978860855103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873554423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173047125339508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873569359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565708577632904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873584157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188698559999466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873599679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491502821445465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873599679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873599680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873643380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7527515888214111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873643381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873643381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784873658639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11722313612699509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873674266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142822876572609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873689246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11739121377468109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873704656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365428030490875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873719508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486963927745819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873734353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089627593755722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873749223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780847072601318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873749224, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873749224, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873791943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7588794231414795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873791944, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873791944, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784873806762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941693931818008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873821633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932140052318573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873836166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457014828920364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873851050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932914167642593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873866306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406384408473969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873881529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383208096027374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873896866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209554970264435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873896867, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873896867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873939054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7624707221984863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873939055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873939055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784873954098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784801840782166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873968842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967233031988144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873984350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12115511298179626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873999726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12226130068302155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874014765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518111824989319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874029978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110188364982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874045125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11640200763940811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874045125, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874045126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874087045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7644136548042297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874087046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874087046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874102058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487240344285965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874117277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799841582775116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874132783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11743854731321335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874148097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106678694486618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874163679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399441212415695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874178887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398621648550034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874194608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447630286216736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874194609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874194609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874237756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661281824111938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874237757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874237757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874253235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849859356880188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874268645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712753236293793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874283988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469253361225128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874299314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358416080474854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874314591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11764125525951385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874329965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625972598791122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874345751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133931577205658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874345751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874345752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874389270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679239511489868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874389270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874389271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874404705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468034237623215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874420453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067500188946724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874435667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814451426267624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874451339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340861022472382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874466497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683618485927582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874481460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354581475257874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874496302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618898808956146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874496302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874496303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874539614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690886855125427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874539614, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874539615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874554611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949137806892395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874569608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487124860286713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874584553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597743093967438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874599445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536263883113861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874614598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205460339784622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874629879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09598654508590698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874645452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544203221797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874645453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874645453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874688675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701836228370667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874688676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874688676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784874703876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508546233177185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874718663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986009240150452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874734499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177596002817154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874749932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096261665225029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874765009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614313930273056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874779983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048998892307281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874795186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617236793041229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874795187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874795187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874837744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711830735206604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874837744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874837745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784874852918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507175326347351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874867686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522839426994324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874882752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943177342414856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874898010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072753444314003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874913426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564559698104858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874928775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117061227560043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874944330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950988531112671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874944330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784874944331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784874988365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716242671012878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784874988366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784874988366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875003255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498504340648651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875018681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016923114657402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875034047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080247238278389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875049489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858781635761261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875064753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471656173467636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875079843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037096589803696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875095073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047038733959198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875095073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875095074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875137503, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722070813179016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875137503, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875137504, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875152594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469478368759155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875167653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09971795231103897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875182663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420377552509308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875198330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992108285427094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875213858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584680736064911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875229655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315334796905518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875245302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095694750547409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875245302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875245302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875287750, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726502418518066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875287750, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875287750, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875302685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953103959560394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875318664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269433259963989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875333583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699674487113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875348553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031702384352684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875363483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225556790828705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875378470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660998523235321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875393530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841883718967438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875393531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875393531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875436173, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729765772819519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875436174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875436174, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875451139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054895669221878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875466199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322723537683487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875480971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779187083244324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875496185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571154952049255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875511449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956151783466339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875526849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728314518928528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875542035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067120730876923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875542035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875542036, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875584941, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733228206634521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875584943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875584943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875599860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684366524219513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875615558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118263214826584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875630817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425108671188354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875646290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173295438289642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875661626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081111878156662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875676932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331483721733093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875692249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800141096115112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875692249, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875692249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875735509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736426591873169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875735510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875735510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784875750712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335630178451538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875765959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083666980266571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875781116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733458399772644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875796470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961307048797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875811980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155813932418823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875827449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225941777229309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875843271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052287220954895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875843271, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875843272, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875885968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737842202186584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875885969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875885969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784875901249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785843431949615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875916088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951336681842804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875932086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546574532985687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875947614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504833400249481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875962860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456966400146484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875978206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358546674251556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875993641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612562298774719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875993642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784875993642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876036723, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736989855766296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876036724, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876036724, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876052156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121830344200134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876067309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332354694604874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876082775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971621423959732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876098520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016611754894257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876114301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393864452838898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876129948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196573078632355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876145919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246471852064133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876145919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876145920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876188846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740312218666077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876188847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876188847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876203895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881702601909637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876219685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721959173679352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876235359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651014000177383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876251304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418122798204422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876266701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11764384806156158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876282174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930554568767548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876298001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11525054275989532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876298002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876298003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876341270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742974758148193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876341271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876341271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876356440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334221065044403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876371958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304493993520737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876387605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554333567619324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876403236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334138363599777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876418955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866191238164902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876434527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871867835521698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876449689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129793524742126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876449690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876449690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876494686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744693756103516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876494687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876494687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876510156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108326755464077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876525639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460807383060455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876540392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416696965694427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876555315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422013700008392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876570163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649580508470535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876585335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026889830827713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876600277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804010927677155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876600278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876600278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876643597, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747345566749573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876643598, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876643598, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876658554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109056755900383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876673323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182122886180878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876688701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725954174995422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876703903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300560295581818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876719246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171197354793549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876734795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528149455785751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876749629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943249821662903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876749630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876749630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876793383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748974561691284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876793383, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876793384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784876807981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772033035755157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876823668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997729003429413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876838951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660609602928162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876854109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596560686826706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876869118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071426272392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876883902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334858298301697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876898929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747091472148895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876898930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876898931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876941630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751222252845764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876941631, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876941631, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784876956705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794425755739212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876971550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502054542303085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876986792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858356952667236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877001893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253095626831055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877017327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847947746515274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877032805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957629978656769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877048510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478397458791733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877048511, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877048511, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877091320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775201141834259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877091321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877091321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877106870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735461115837097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877121786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175085812807083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877137265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208415776491165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877152061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219833999872208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877167721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989409148693085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877182878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237956047058105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877197914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053721159696579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877197914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877197915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877241608, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753732204437256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877241609, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877241609, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877256426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102993443608284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877271828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002323985099792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877287380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841314494609833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877302784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942204296588898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877317911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042664498090744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877333272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809394717216492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877348779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091051697731018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877348780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877348780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877391602, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756937742233276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877391603, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877391603, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877406997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108466237783432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877422814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406004637479782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877438354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043575182557106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877453411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537479817867279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877468775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991897642612457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877484136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716941207647324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877499942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371799767017365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877499943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877499943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877543867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757908701896667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877543867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877543868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877559226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062736064195633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877574543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755100846290588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877590004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000943392515182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877605673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088067889213562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877621196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09664613008499146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877636809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174892842769623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877652023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381093621253967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877652024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877652024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877696893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759956121444702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877696893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877696894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784877712517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213927924633026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877728232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771737992763519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877743852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828986763954163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877759761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104770302772522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877775477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760228335857391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877790668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908931493759155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877806313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133074223995209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877806314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877806314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877851807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77634197473526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877851808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877851808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784877866917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943952202796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877882308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381582379341125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877897343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668186843395233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877912410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139500141143799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877927525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754232108592987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877942635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225109547376633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877957582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351458191871643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877957582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784877957583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878002338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765970230102539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878002339, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878002339, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878017117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060328558087349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878032079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777149558067322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878047070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009465903043747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878062209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435119271278381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878077597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125467717647552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878092977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212539672851562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878108876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349702090024948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878108877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878108877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed151156919.log b/recommendation_v4/rcp_logs/gbs_16384/seed151156919.log new file mode 100644 index 000000000..59dab1901 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed151156919.log @@ -0,0 +1,799 @@ +:::MLLOG {"namespace": "", "time_ms": 1784846779600, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784846779600, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784846806661, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "seed", "value": 151156919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784846806663, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784846807048, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784846807048, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784847733125, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784847733126, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784847733411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784847862224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882054388523102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847876372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388241946697235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847890057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869300484657288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847903933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871368765830994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847917861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874506950378418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847931753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386300027370453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847945559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138418048620224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847945560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847945560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847988754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49982750415802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847988755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847988755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784848002616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874763250350952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848016385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869933784008026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848030442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869796693325043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848044440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386922150850296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848058476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865910470485687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848072840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384579986333847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848087321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885018229484558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848087322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848087323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848130925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49998733401298523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848130925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848130926, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784848145013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384497433900833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848159586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827064633369446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848173560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382831335067749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848188192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382591724395752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848202170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827215135097504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848216759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384887844324112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848230639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380145400762558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848230639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848230640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848272923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001637935638428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848272924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848272924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784848287006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833743333816528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848301871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800093531608582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848316130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741961121559143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848330491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382007896900177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848345715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785354793071747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848359932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766677677631378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848374691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374267339706421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848374692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848374692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848416800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003942251205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848416801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848416801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784848430564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708099722862244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848444678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767597079277039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848459263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756930828094482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848473313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371050775051117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848487568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359502673149109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848501811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371367871761322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848516152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360795497894287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848516153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848516154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848559407, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006735324859619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848559408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848559408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784848573471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694435358047485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848587453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646936416625977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848601920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757139444351196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848616569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664746284484863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848630847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514992594718933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848645205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355534791946411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848659687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748618960380554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848659688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848659689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848704803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009748935699463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848704803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848704803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848719544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685576617717743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848734222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357807070016861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848748423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600876927375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848762994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351243555545807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848777671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518376648426056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848792959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476210832595825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848807845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513348996639252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848807845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848807846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848850798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013629794120789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848850799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848850799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848865638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425996899604797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848880544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363610476255417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848895991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138233482837677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848910739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416802883148193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848925714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13612619042396545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848940435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13491898775100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848955669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366947591304779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848955670, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848955670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848999645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019301176071167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848999646, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848999646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784849014733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366378664970398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784849029755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366163194179535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784849044609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477492332458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784849059411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050410151481628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784849074393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265976309776306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849088748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337883174419403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849103050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348382532596588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849103103, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849103103, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849146667, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025563836097717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849146668, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849146668, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784849161162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349460482597351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849175483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938405573368073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849190796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314333826303482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849205128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368253380060196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849219634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229815661907196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849234091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157176971435547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849249323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253915309906006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849249324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849249324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849293509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032495260238647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849293510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849293510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784849307673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317821741104126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849322985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425028324127197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849337128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279902160167694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849351222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12965713441371918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849365565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377517461776733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849380055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12819096446037292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849394126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978589534759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849394127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849394127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849438364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041555166244507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849438364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849438364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784849453527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466337323188782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849467861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308776915073395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849482531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345287263393402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849497107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349264681339264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849512195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362331211566925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849526770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368327915668488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849541612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697633147239685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849541613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849541613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849585132, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055156350135803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849585133, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849585133, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784849599948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432535529136658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849614902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306743025779724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849629730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383376598358154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849644854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403389871120453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849659770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304193139076233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849675100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324896663427353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849690301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818489015102386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849690301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849690302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849733240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070247650146484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849733241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849733241, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784849748533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263432502746582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849763635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313425898551941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849778943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705787062644958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849793996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535504043102264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849809377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244761526584625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849823789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12867212295532227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849838396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12803372740745544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849838397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849838397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849881689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5085452198982239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849881689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849881690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784849896373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003361225128174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849911066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489823043346405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849925968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1422210931777954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849940597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13405194878578186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849955962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314431130886078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849970855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363401591777802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849985733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600751757621765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849985733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784849985734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784850029356, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5101237893104553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784850029357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784850029358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784850044055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346210092306137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850058961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629218935966492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850074164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564729690551758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850089170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314875960350037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850104393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343318074941635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850119384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329774558544159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850134051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255181908607483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850134052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850134052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850178689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5119063258171082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850178690, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850178690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784850193342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196393847465515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850208174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345001608133316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850223154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14192607998847961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850238334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363781571388245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850253365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134255051612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850268624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234907388687134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850283567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331523209810257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850283568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850283568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850327370, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5139719843864441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850327370, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850327371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784850342164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383851021528244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850356979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314425945281982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850371724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287626326084137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850386567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300586611032486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850401790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793018460273743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850417028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163718581199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850432148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621094822883606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850432149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850432149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850477299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5163280367851257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850477299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850477300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784850492062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683071732521057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850506541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643181324005127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850521180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139963150024414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850535602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013362884521484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850550255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157643377780914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850565160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354098618030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850580113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326732635498047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850580113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850580114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850623105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5189617872238159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850623106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850623106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784850637813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592734932899475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850653494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231578469276428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850669038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782858848571777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850684344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362910270690918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850699773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859358429908752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850715140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444578647613525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850730834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391860395669937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850730835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850730835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850774938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.521634578704834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850774939, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850774939, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784850790451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501930236816406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850805982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14851126074790955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850821824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521283864974976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850837017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311710953712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850852267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13427919149398804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850867271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332881599664688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850882552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365194320678711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850882552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850882553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850925433, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5246269106864929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850925434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850925434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784850940728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606123626232147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850955838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526341319084167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850971290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389819860458374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850986213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336824595928192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851001205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333129107952118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851016122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295630782842636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851030768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12569110095500946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851030769, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851030769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851073463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5280777215957642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851073464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851073464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784851088160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275770843029022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851103069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257047533988953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851118035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504862785339355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851132982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353694051504135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851148272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678652048110962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851163142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12887413799762726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851177821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977521121501923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851177822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851177823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851221279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5330978035926819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851221280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851221280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784851235815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256460428237915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851250311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706884741783142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851264940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331857144832611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851279651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218916952610016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851294611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271083652973175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851309807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236582279205322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851324424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444076478481293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851324424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851324425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851368533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5396130681037903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851368533, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851368533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784851384212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294931173324585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851399352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052934408187866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851414222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070666790008545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851429177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12993022799491882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851443907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12793771922588348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851458773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12336001545190811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851474043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273115575313568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851474044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851474044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851517330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5484501719474792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851517330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851517330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784851532512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588634133338928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851547661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335800170898438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851562310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922393321990967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851577879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373517990112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851592724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12794984877109528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851607513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298736333847046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851622494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127152681350708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851622494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851622495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851666607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5593397617340088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851666608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851666608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784851681083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12908676266670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851695493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12775126099586487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851710247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280647993087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851725114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283148407936096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851740230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316329389810562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851755005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340041309595108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851770360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13068073987960815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851770361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851770361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851813972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5720669627189636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851813973, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851813973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784851829216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996506690979004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851844224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992912530899048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851859230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301020085811615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851874353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299424946308136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851889217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13400772213935852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851903813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300612986087799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851919176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081389665603638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851919177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851919177, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851962097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5847154259681702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851962097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851962097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784851977150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870632112026215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851992665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13146652281284332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852007491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911540269851685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852023200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282547414302826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852038434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799251079559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852053540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242407143115997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852068785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324198126792908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852068786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852068786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852112188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5967762470245361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852112188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852112188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784852127040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13071003556251526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852142418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288137435913086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852158153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127401471138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852173948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248059153556824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852189610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398823142051697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852204849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010163605213165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852220868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944281101226807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852220869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852220869, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852264888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6088135838508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852264889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852264889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784852280176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072431087493896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852295481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861061096191406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852310841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13022005558013916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852326042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362059861421585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852341462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150455057621002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852356999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12708717584609985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852373470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096663355827332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852373471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852373471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852417312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6180189251899719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852417312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852417312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784852432421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745684385299683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852448380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290859878063202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852463666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128651961684227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852478733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290985643863678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852493794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12663903832435608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852508901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398587912321091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852523920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12526993453502655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852523921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852523921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852567398, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6279645562171936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852567399, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852567399, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784852583037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12855257093906403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852598515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12444411218166351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852613845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11909816414117813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852629075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12448503077030182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852644290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12055952101945877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852659423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12134136259555817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852674606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894641816616058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852674607, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852674608, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852718131, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6375717520713806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852718132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852718132, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784852733408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236836239695549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852748285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199759691953659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852763306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727391183376312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852778374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12392610311508179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852793838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12194432318210602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852809331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043236196041107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852824410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121465265750885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852824411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852824411, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852868306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.65995192527771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852868306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852868307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784852883253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822301149368286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852898856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791133135557175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852914082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12175138294696808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852929099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11661405861377716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852945122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150299459695816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852960101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652299016714096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852975256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123342365026474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852975317, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784852975318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784853019462, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6770018935203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784853019463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784853019463, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784853034380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769773066043854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853049985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552567780017853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853065466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977910250425339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853081362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115785151720047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853096489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573126912117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853112184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330823391675949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853127201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102057546377182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853127202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853127203, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853170667, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7059394121170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853170668, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853170668, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784853185703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792708933353424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853200788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159525066614151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853216079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380937695503235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853231152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235920339822769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853246207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769937932491302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853261089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022453546524048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853276691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644601076841354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853276692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853276692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853320740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7365131974220276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853320740, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853320740, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784853336123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11910886317491531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853351851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595627665519714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853367058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178789958357811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853382699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490047186613083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853398216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624341458082199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853413800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112409830093384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853429129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965251922607422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853429129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853429130, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853472481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.755134642124176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853472481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853472481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784853487579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111704982817173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853502869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114948987960815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853518030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526723951101303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853533525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090824007987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853549224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539767682552338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853564768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549770295619965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853580218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281201988458633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853580219, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853580219, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853624312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.762135922908783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853624312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853624313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784853639825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867486894130707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853654933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093977093696594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853671083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12095609307289124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853687000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12323690950870514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853702604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714081466197968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853718331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188685148954391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853734055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665312200784683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853734056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853734056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853778154, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766153872013092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853778155, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853778155, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784853793781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630232632160187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853809611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091659814119339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853825642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831356585025787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853841488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137460172176361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853857651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488417536020279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853873293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444784700870514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853889720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535600781440735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853889721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853889721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853935012, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679151892662048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853935013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853935013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784853950601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189681813120842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853966241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803516209125519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853981860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056821271777153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853997333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414182186126709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854012718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906518042087555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854028128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744224488735199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854043951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470820009708405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854043952, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854043953, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854089104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687287926673889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854089105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854089105, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784854104744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554098129272461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854120691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791954398155212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854136010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10902760922908783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854152142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377592384815216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854167610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771903395652771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854182952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449446737766266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854198213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720092594623566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854198214, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854198214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854242675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768823504447937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854242676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854242676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784854258026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945896804332733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854273392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498135328292847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854288709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681477189064026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854303838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503461956977844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854319483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255461931228638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854335018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621278196573257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854350731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574954748153687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854350732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854350732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854394711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769791305065155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854394712, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854394712, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784854410040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559000074863434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854425082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070096492767334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854441114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11885470151901245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854456482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952217876911163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854471669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643278062343597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854486602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008431613445282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854501975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625860095024109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854501975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854501976, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854545738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705718278884888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854545739, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854545739, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784854561001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534784942865372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854575958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583368688821793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854591098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943309962749481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854606376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739423334598541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854621851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632839798927307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854637238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196541786193848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854652808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968375205993652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854652809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854652809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854697048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713091373443604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854697049, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854697049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784854712218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467244684696198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854727783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241571813821793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854743349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881274938583374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854758929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882112383842468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854774299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049659475684166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854789692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064209997653961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854805343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516241937875748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854805344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854805344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854849231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718011140823364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854849232, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854849232, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784854864751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450093448162079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854880350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017445683479309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854895886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483947396278381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854911817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029823124408722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854927656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596564412117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854943756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348650813102722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854959594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980327427387238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854959594, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784854959595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784855003259, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722395062446594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784855003260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784855003260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784855018402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022804141044617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855034397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030786782503128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855049787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684189200401306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855065178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357688367366791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855080427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281796753406525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855095716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665173083543777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855111126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973069071769714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855111127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855111127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855154207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772571325302124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855154208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855154208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784855169413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534993559122086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855184842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338887572288513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855200065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820692032575607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855215673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577557235956192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855231391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992462933063507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855247219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757985711097717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855262833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083436012268066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855262834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855262834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855306858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727365493774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855306858, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855306859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784855322006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693249106407166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855338105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163913667201996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855353712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458188503980637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855369650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211275517940521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855385376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848505795001984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855401129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352227628231049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855416861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826694220304489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855416861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855416862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855460142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730767130851746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855460143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855460143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784855475791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375536978244781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855491257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135101318359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855506836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773605108261108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855522598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004182159900665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855538541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177179217338562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855554414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284411698579788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855570610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500927269458771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855570610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855570611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855613657, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773326575756073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855613658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855613658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784855629273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082344651222229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855644543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016219317913055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855660723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595264077186584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855676616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557774990797043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855692363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482610553503036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855708103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390865802764893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855724036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657839477062225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855724037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855724037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855766585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733494639396667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855766586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855766586, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784855782506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105512082576752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855798235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399313807487488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855814031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995682328939438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855830030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021818220615387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855846011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463446915149689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855861868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198095232248306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855877864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307837069034576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855877865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855877865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855920679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773645281791687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855920679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855920680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784855935912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929356515407562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855952057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710210353136063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855968030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732079297304153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855984201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146358847618103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856000150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11858002841472626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856016245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986648499965668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856032591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544212698936462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856032592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856032592, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856077589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738677263259888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856077590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856077590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784856093493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11335670202970505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856109548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135437935590744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856125791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580827832221985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856142061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135413646697998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856158295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943345725536346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856174376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889366269111633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856190026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247888952493668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856190026, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856190027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856235695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739756107330322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856235696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856235696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784856251719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894247144460678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856267548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495366156101227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856282666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423135757446289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856297958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431760549545288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856313168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667216777801514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856328729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294046252965927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856344088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820260643959045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856344089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856344089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856388384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740434408187866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856388384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856388384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784856403684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094837486743927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856418818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242711007595062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856434336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767830908298492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856449813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335372388362885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856465464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213589668273926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856481258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598452389240265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856496586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993871301412582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856496587, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856496587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856542188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742499709129333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856542188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856542189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784856557093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787840187549591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856573231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019475758075714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856588877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065472662448883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856604400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632923245429993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856619972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734912753105164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856635263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032852753996849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856650819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792672634124756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856650820, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856650820, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856694841, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743613123893738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856694841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856694842, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784856710204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789279639720917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856725485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586035996675491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856741082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874854773283005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856756557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280495136976242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856772484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881105065345764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856788410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954567790031433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856804314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050684005022049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856804315, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856804315, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856848568, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744684815406799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856848569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856848569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784856864503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792002081871033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856879733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201144963502884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856895515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214528232812881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856910380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272318124771118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856926023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054314881563187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856941299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298210382461548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856956440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054534912109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856956441, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784856956442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784857000403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745795845985413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784857000403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784857000404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784857015610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302111506462097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857031252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10049617290496826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857046831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821403563022614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857062238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961610078811646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857077571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445409268140793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857093248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797997564077377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857108936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922190546989441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857108936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857108937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857151728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745900750160217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857151729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857151729, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784857167421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105247288942337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857183224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458701103925705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857198655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446542501449585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857213662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578341037034988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857229427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011807829141617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857245240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758892446756363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857261127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436390340328217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857261127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857261128, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857304449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746885418891907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857304449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857304449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784857319829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704543441534042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857335099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829165577888489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857350801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000364482402802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857366272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940422862768173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857381680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671854972839355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857397204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253430902957916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857412375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385517030954361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857412375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857412376, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857456258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748563885688782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857456259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857456259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784857471912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102349191904068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857487490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852664709091187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857503074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901114344596863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857518824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108499765396118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857534407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799803584814072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857549602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909239947795868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857565447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148080438375473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857565448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784857565448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784857609333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748023271560669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784857609334, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784857609334, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed169450858.log b/recommendation_v4/rcp_logs/gbs_16384/seed169450858.log new file mode 100644 index 000000000..e3b7195b5 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed169450858.log @@ -0,0 +1,1378 @@ +:::MLLOG {"namespace": "", "time_ms": 1784704997579, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784704997579, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784705022955, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "seed", "value": 169450858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784705022957, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784705028098, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784705028099, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784705672887, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784705672888, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784705673138, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784706124390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13934564590454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706137940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909479975700378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706151384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881759345531464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706164984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876499235630035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706178846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864529132843018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706192588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866840302944183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706206248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390020251274109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706206248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706206249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706293520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012962222099304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706293521, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706293521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706307025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899950683116913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706320603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885724544525146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706334313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877752423286438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706348206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881084322929382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706362038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874900341033936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706376367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389700472354889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706390758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388654112815857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706390759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706390759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706434053, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014854073524475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706434054, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706434054, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706447926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870179653167725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706462076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848313689231873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706475794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849811255931854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706490100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856196403503418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706503663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855358958244324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706517913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385328471660614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706531840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837596774101257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706531841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706531841, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706575410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017085075378418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706575411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706575411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706589334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851556181907654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706603699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384570300579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706617524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781607151031494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706631615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829545676708221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706646063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380615532398224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706659917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771703839302063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706674352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785254955291748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706674353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706674353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706718015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020002126693726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706718016, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706718016, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706731544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741199672222137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706745303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776597380638123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706759274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768726587295532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706773027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745611906051636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706786967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633498549461365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706800897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720612227916718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706814894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364779770374298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706814895, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706814896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706859747, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023314356803894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706859748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706859748, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706873385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370897889137268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706886943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678543269634247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706900978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760198652744293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706915119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696140050888062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706928866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547061383724213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706942653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359313726425171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706956593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137655571103096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706956593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706956594, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784707001910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026870965957642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784707001911, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784707001911, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784707015878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705486059188843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707029899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359308660030365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707043699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362440586090088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707057656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541041314601898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707071734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550639152526855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707086169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496294617652893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707100265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521766662597656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707100266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707100266, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707143986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031247735023499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707143986, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707143986, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707158121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453572988510132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707172414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660217821598053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707187024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811524212360382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707201378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443204760551453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707215974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611042499542236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707230301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515549898147583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707244967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652117550373077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707244968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707244968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707289003, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037915110588074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707289003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707289003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707303244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651007413864136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707317453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663959503173828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707331659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498686254024506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707345925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073977828025818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707360382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294528424739838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707374568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388067483901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707388720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493485748767853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707388771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707388771, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707432006, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045122504234314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707432007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707432007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707446351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350564956665039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707460565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960085272789001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707475287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152998685836792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707489024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671524822711945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707502972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325109899044037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707516729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317097544670105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707531096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264797627925873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707531097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707531097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707573809, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053156018257141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707573809, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707573810, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707587501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197173178195953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707602200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426119089126587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707616112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829650938510895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707630223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298401653766632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707644567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375365734100342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707659149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12844517827033997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707673161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001719117164612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707673161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707673162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707717168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064457058906555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707717168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707717169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707731808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462510704994202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707745724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330670565366745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707759892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438847661018372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707773945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348436176776886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707788547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599160313606262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707802776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363826274871826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707817149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692057132720947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707817149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707817150, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707860810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5079631209373474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707860811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707860811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707875347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342317909002304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707890059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052499294281006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707904577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339837908744812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707919475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14027121663093567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707934111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130463108420372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707948794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246062397956848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707963049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381942182779312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707963050, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707963050, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708004955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5094265341758728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708004956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708004956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708019472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257256269454956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708033830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125458359718323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708048664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700002431869507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708063216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541248440742493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708078082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236276805400848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708092230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288347840309143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708106551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796840071678162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708106551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708106552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708149231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5108858942985535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708149231, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708149231, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708163310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013429939746857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708177705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474637269973755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708192240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14225800335407257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708206593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415345549583435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708221508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145878911018372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708235870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336643099784851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708250503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605180382728577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708250503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708250504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708294031, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125228762626648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708294032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708294032, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708308417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462844491004944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708322790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362040638923645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708337500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577982783317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708352105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324254751205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708366699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435284793376923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708381219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330452263355255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708395574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257843255996704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708395574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708395575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708438185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.514238715171814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708438185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708438186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708452523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196463882923126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708466940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345776915550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708481449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14172695577144623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708496402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410049676895142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708511165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343328058719635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708525891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242995738983154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708540353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312357664108276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708540354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708540354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5161473751068115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583011, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708597095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384182870388031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708611259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298577070236206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708625305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288609504699707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708639435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990142405033112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708653845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379750669002533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708668452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168470561504364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708682707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624517619609833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708682708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708682709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708727487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182508230209351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708727487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708727487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708742010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136775940656662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708756174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364579200744629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708770670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314641833305359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708784854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006696105003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708799290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144081830978394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708813948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534171879291534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708828703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13274559378623962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708828703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708828704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708872585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5205264091491699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708872585, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708872585, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708887047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600826263427734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708902030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228470087051392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708916831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781657814979553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708931481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637033104896545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708946172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852858543395996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708960645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450099527835846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708975623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391758918762207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708975624, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784708975624, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709018913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5227922797203064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709018914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709018914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709033831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497819006443024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709048738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14756529033184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709063904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514935970306396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709078808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299930095672607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709093606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342543661594391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709108164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301649689674377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709122818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665775954723358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709122819, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709122819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709165957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5252751708030701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709165958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709165958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709180783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361621916294098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709195674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535308837890625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709210573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377340137958527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709224981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367971777915955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709239371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329064846038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709253780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294342577457428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709267972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12586551904678345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709267973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709267973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709311580, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5281782150268555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709311580, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709311580, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709325946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271589577198029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709340588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266083598136902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709355348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350477933883667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709370092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541118800640106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709384794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369103193283081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709399217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289854645729065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709413465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001489639282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709413466, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709413467, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709457135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5322018265724182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709457136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709457136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709471393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276158273220062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709485561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273319125175476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709499842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318373262882233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709514344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237938284873962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709529161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726779282093048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709544198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242024183273315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709558549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345301866531372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709558550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709558550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709602167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5373883247375488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709602168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709602168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709617115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12967930734157562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709631570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13048714399337769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709645882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091596961021423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709660352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009649515151978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709674468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811893224716187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709688721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12348876893520355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709703489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750403583049774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709703489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709703490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709746223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5443886518478394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709746224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709746224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709760786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598410785198212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709775440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379690051078796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709789543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955292105674744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709804626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397543132305145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709819508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281648576259613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709834417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017219305038452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709849498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12759943306446075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709849498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709849499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709892137, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5534690618515015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709892138, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709892139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709906679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964753806591034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709921382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281525045633316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709936347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330098658800125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709951330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334733247756958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709966318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203084468841553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709981008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344614177942276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709995883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107417523860931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709995883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784709995884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710038461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5655330419540405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710038461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710038461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710053164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017848134040833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710067870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13048714399337769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710082643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13074448704719543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710097334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357073068618774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710111899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463521003723145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710126165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130702942609787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710141299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138309121131897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710141299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710141300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710184491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5791877508163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710184492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710184492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710199295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129245787858963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710214471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225261867046356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710228903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995046377182007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710244183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911981344223022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710259770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12874147295951843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710275106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330917328596115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710290546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338033676147461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710290546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710290547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710333581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5925002098083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710333581, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710333581, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710348489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152767717838287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710363583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295604407787323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710379073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198989629745483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710394462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309034705162048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710409950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465771079063416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710424983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073796033859253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710440393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300548017024994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710440393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710440394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710483451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6048631072044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710483451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710483451, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710498241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127043843269348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710513328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12942911684513092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710528544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098296523094177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710543383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663187623023987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710558611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205131888389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710573906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753908336162567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710589217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170988857746124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710589217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710589218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710632867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6142460107803345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710632867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710632867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710647652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282360702753067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710663279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958259880542755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710678376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12888579070568085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710693391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929807603359222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710708162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12710097432136536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710723112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12467522919178009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710738018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12523584067821503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710738019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710738019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710780385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.622561514377594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710780386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710780386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710795965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893664836883545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710811248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470994889736176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710826393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906374990940094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710841440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12448423355817795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710856381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207442432641983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710871062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12135551869869232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710885916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189020574092865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710885917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710885917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710928793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6318137049674988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710928794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710928794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710943824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12405118346214294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710958431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119718998670578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710973045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617644876241684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710987834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12368768453598022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711002893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12102761119604111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711018157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201084554195404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711033035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112934798002243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711033036, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711033037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711076911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6459285020828247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711076912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711076912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711091511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11756554245948792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711106641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11745916306972504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711121566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12033573538064957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711136213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576130986213684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711151076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137530729174614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711165764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535854637622833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711180290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117687001824379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711180333, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711180334, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711224096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6601696610450745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711224097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711224097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711238561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11647741496562958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711253832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438675969839096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711268831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870233714580536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711284199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457877606153488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711298902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468038707971573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711314190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12129223346710205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711328897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904015600681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711328897, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711328898, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711373232, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6828247904777527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711373233, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711373233, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711387936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649450659751892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711402659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604668200016022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711417526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266785860061646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711432048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212901771068573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711446617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650745034217834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711460900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1206890195608139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711476207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618584394454956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711476208, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711476208, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711519745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7120975255966187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711519746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711519746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711534546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884923279285431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711549730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623626947402954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711564452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831817030906677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711579586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449458658695221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711594621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593669652938843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711609289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040298640727997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711624006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937277972698212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711624007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711624007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711668242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7349262237548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711668243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711668243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711682901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137908697128296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711697583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116205155849457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711711987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11653327941894531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711726653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148111522197723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711741747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638261377811432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711756621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688932240009308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711771539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148272454738617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711771540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711771540, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711815375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7484737038612366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711815375, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711815375, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711830258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949403047561646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711844787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118779182434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711860087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12250057607889175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711875216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234634667634964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711890026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11788441985845566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711905094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237897723913193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711920199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814898252487183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711920199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711920200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711963263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7553998827934265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711963263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711963264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784711978122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700116097927094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711993262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979719460010529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712008769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11862444877624512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712024076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272574216127396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712039702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115536168217659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712054825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524207144975662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712070181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534532368183136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712070181, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712070182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712114233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7599262595176697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712114234, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712114235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712129376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889444291591644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712144463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857857018709183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712159472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687479376792908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712174329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11513744294643402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712189202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889898031949997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712204235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739222168922424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712219673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465433239936829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712219673, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712219674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712262020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7624871134757996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712262020, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712262020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712277150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560891568660736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712292640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767362266778946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712307569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912342369556427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712322931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424178630113602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712338171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769177973270416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712353115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403800010681152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712367963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604955792427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712367964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712367964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712410463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7642265558242798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712410464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712410464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712425305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065420508384705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712440190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621294915676117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712454944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652686655521393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712469574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552902519702911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712484678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031557098031044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712499745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597709774971008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712515258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581743717193604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712515259, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712515259, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712559691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7644188404083252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712559691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712559691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712574511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517622530460358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712589131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994886606931686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712604635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11806486546993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712619834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960260033607483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712634691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652033984661102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712649406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008707731962204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712664382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609720647335052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712664383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712664383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712707729, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656647562980652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712707730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712707730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712722731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054212898015976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712737412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569630563259125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712752306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996456444263458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712767421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738703608512878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712782594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632451623678207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712797761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153200268745422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712813136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025360226631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712813136, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712813137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712857279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766525387763977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712857279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712857280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712872160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499867051839828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712887279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244914889335632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712902292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864590108394623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712917444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833898931741714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712932368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050087958574295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712947311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078812390565872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712962212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523165762424469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712962212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784712962213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713005771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670781016349792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713005772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713005772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713020627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521787405014038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713035404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964337944984436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713050067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517880320549011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713065318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106695979833603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713080446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625691711902618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713095913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354968160390854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713111221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10902467370033264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713111221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713111222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713155239, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680963277816772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713155239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713155240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713169881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940485656261444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713185327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285056382417679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713200522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765117406845093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713215629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380882024765015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713230616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298453271389008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713245685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745249688625336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713260958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921622067689896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713260958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713260959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713303771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690072059631348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713303772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713303772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713318780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565415024757385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713333765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341772437095642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713348692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842376202344894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713364190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620873421430588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713379742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984630137681961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713395317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786747932434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713410762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072956025600433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713410763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713410763, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713453832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699215412139893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713453833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713453833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713468743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690926015377045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713484202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017439067363739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713499459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504136234521866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713514779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204260051250458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713529876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089305728673935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713544953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338037252426147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713560062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086040586233139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713560062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713560063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713602755, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770125150680542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713602755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713602755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713617876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379593819379807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713632910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135870218276978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713648020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725864768028259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713663249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997251495718956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713678477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207271367311478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713693754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274390667676926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713709526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611146688461304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713709527, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713709527, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713752367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702265977859497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713752368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713752368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713767562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822077095508575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713782523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013392567634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713798420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581400036811829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713814219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523501574993134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713829696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491701006889343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713845147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401125252246857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713860807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659560561180115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713860807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713860808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713903690, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770200252532959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713903691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713903691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784713919275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146437376737595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713934626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398723721504211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713950273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008527874946594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713966096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035259068012238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713981884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146092563867569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713997509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242333054542542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714013403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126490980386734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714013404, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714013405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714057295, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77055823802948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714057295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714057296, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714072542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918173938989639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714088268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737434774637222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714103704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654455423355103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714119557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508001387119293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714135027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11840542405843735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714150535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968900471925735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714166529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554568260908127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714166529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714166530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714210278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711919546127319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714210278, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714210279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714225729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364011466503143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714241293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364781111478806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714256915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11551962047815323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714272558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355239897966385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714288334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905277729034424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714303775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896290838718414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714318831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190667748451233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714318832, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714318832, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714363049, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715903520584106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714363050, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714363050, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714378599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880498588085175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714394239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046888679265976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714409142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506308078765869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714424409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464328527450562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714439490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672253370285034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714454825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362933576107025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714469961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831527411937714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714469962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714469962, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714514237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721980214118958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714514237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714514237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714529380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918289422988892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714544352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227397084236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714559648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773412883281708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714574991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335443168878555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714590625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184946656227112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714606276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557729750871658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714621524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997723862528801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714621524, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714621525, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714664950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722623944282532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714664950, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714664950, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714679972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779687762260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714695592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105005294084549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714711221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659851878881454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714726269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599905252456665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714741284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782286524772644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714755988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379375517368317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714771037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792653262615204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714771037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714771038, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714814697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726943492889404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714814698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714814698, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784714829772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781963169574738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714844594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582665354013443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714859853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896560549736023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714874984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263598710298538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714890370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900495201349258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714905860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970762372016907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714921421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512566566467285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714921421, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714921422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714963801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772916316986084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714963801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714963802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784714979244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791385173797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714994245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236972570419312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715009539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206050217151642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715024478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342784225940704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715040061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014945268630981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715055167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246367752552032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715070018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628064721822739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715070018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715070019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715111978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732444405555725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715111978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715111979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715126679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331086814403534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715141866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10067000985145569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715157210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847043991088867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715172265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966414213180542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715187250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442395508289337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715202510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831081867218018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715217970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954949259757996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715217971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715217971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715261313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735998630523682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715261314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715261314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715276816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100797355175018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715292561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486466437578201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715307885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435252636671066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715322760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589651763439178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715337986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038968920707703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715353067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724781453609467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715368391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439062118530273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715368392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715368392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715411910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737373113632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715411911, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715411911, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715426887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067752093076706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715441719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790053009986877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715456666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991295635700226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715471933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092996746301651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715486940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09668903052806854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715502016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249090194702148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715516802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038886308670044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715516802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715516803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715560752, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737910151481628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715560753, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715560753, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715576054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225589573383331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715591241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880200564861298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715606561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880531370639801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715622059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159330606460571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715637449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795804858207703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715652321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937802493572235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715667651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168161988258362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715667651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715667651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715711632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742061018943787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715711632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715711633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715726582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936897993087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715742157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104078508913517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715757311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677896440029144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715772565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173360794782639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715787778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779348760843277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715803284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301404237747192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715818677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040080338716507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715818678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715818678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715863718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741485834121704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715863719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715863719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784715879005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678361356258392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715894327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09820735454559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715909702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096575856208801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715925278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439030826091766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715940986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184456944465637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715956834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112368144094944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715972816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397449135780334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715972817, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784715972817, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716016461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774474024772644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716016462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716016462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716032019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277226567268372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716047046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708660632371902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716062928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250790417194366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716078442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033092439174652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716094465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099572479724884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716110054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854058504104614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716125444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771951615810394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716125444, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716125445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716169161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746531367301941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716169161, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716169161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716184689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11839602887630463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716200319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976830869913101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716215730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150747761130333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716231204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191844195127487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716246856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473256349563599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716262685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165887862443924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716278423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891944169998169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716278424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716278424, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716322988, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744470834732056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716322989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716322989, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716338702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149198561906815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716354487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046934574842453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716369847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245617270469666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716385994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179967224597931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716401997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030451953411102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716417633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784877836704254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716433201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017562448978424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716433202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716433202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716476165, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774411678314209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716476166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716476166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716491527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355479061603546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716507035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520391911268234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716522832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821182280778885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716538305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944776237010956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716553892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484190821647644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716570103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216242611408234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716585740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587244272232056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716585740, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716585741, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716628989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746368050575256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716628989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716628990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716645177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11588144302368164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716660978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487625539302826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716676302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11684540659189224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716691977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424716770648956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716707402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10153334587812424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716722701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640478879213333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716737875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628598928451538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716737875, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716737876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716780619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748202681541443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716780620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716780620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784716795629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043190956115723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716810892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075211763381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716826124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015079915523529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716841451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047772541642189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716857121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440938174724579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716872247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0978470891714096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716887726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055726483464241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716887726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716887727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716930427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749550342559814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716930428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716930428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784716945997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623591393232346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716961829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09950584173202515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716977586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09436272084712982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716993151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10044125467538834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717008318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285276710987091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717023702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562553256750107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717039056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985551029443741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717039057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717039057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717081422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775263786315918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717081423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717081423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717097385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610409080982208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717112788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304738581180573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717127997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11596254259347916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717143337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09791189432144165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717158826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071079820394516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717174138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428725183010101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717189518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007135808467865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717189519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717189519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717232888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754648923873901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717232889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717232889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717248277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036969184875488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717263729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105600506067276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717279532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598866641521454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717295242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752294957637787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717311082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285156965255737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717326843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326234996318817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717342441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563100129365921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717342442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717342442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717386607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755644917488098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717386608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717386608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717401580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250881314277649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717417182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058211401104927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717432385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285107791423798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717447851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117731034755707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717463087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706163942813873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717478244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294287651777267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717493643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348843574523926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717493644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717493644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717536589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758268713951111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717536590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717536590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717551924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251526534557343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717567203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297829657793045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717582381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024412721395493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717597480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295794904232025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717612986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597911477088928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717628653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014921218156815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717644225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897997975349426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717644225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717644226, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717687606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758817076683044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717687607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717687607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784717703141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118260979652405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717718704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902419149875641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717734017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052886605262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717748862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09835644066333771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717764620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619896650314331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717779847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482576489448547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717795226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709483176469803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717795227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717795227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717838318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762871384620667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717838319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717838319, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784717853527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586218535900116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717868592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903058409690857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717883704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503502190113068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717899080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09971314668655396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717914242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966272115707397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717929435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404357314109802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717944448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430090874433517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717944448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784717944449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784717987838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764149308204651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784717987839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784717987839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718003122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09829320013523102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718018606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700777918100357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718033909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237913578748703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718049715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075074225664139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718065336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794331878423691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718080815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114858090877533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718095983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371427237987518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718095983, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718095983, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718140903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776760458946228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718140903, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718140903, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718156335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119932889938354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718171620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998442769050598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718187560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105492174625397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718203030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564713925123215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718218473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037228465080261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718233795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111309677362442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718249168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961611032485962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718249168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718249169, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718293175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769425511360168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718293175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718293175, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718308764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532509535551071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718324200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224469751119614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718339398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727985203266144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718354809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369716584682465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718370344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715292394161224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718386140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128745898604393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718401734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237938165664673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718401734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718401735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718444633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777364194393158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718444634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718444634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718460415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655859112739563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718476205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085964947938919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718491693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258321464061737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718507094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659569501876831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718522633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776382684707642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718538244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077551543712616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718554016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553133279085159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718554016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718554017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718598156, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778531312942505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718598157, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718598157, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718613760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600222647190094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718629271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817550122737885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718644930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268860101699829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718660645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439554393291473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718676342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901325941085815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718691996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614500939846039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718707633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687576979398727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718707634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718707634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718752205, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779173254966736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718752205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718752206, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784718767863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117085069417953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718783607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112609505653381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718799491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052393913269043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718815704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241906881332397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718831759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913659632205963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718847236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420922726392746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718862821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346678644418716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718862822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718862822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718906666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778179407119751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718906667, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718906667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784718922384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808008909225464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718938749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111049354076385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718954575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688400268554688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718970371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284898221492767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718986106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407589912414551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719002145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905386507511139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719017917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355520784854889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719017918, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719017918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719060708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77876877784729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719060709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719060709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719076306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437833309173584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719092289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866573452949524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719108297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305172741413116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719124260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123690009117126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719140358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719306766986847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719156302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512807220220566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719171837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035403609275818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719171837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719171837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719215917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790215611457825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719215917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719215917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719231665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021120309829712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719246916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065647304058075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719262565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286694020032883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719277686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056973934173584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719292819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467492043972015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719307900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144206672906876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719323199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815308451652527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719323200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719323200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719366672, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795535326004028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719366673, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719366673, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719381850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016518086194992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719397021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411946475505829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719412120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137036442756653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719427232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09724283963441849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719442503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112623870372772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719457675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018562912940979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719473120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478146374225616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719473121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719473121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719516827, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795337438583374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719516827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719516828, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719532392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829724371433258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719547752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236680507659912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719563126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09312150627374649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719578218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415370017290115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719593914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166840255260468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719609238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194624960422516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719624944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584545135498047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719624945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719624945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719668787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802691459655762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719668788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719668788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784719684207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09762443602085114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719699602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538434982299805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719714686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848619043827057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719730119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904220700263977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719745700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159961879253387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719760826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021275445818901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719776108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037774384021759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719776109, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719776109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719820693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804332375526428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719820694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719820694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784719835794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188956558704376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719851481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896667093038559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719866993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09611663222312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719882202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328197479248047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719897899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297082364559174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719913767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834529459476471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719929232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598879307508469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719929232, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719929233, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719973572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810023427009583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719973573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719973573, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784719988951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181226581335068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720004046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462918877601624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720019637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589256137609482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720035280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012894868850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720050955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591012239456177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720066524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075468584895134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720081697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025223582983017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720081698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720081698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720125432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810301184654236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720125433, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720125433, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720140913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393649339675903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720156433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110801458358765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720172191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115319490432739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720187907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922226518392563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720203492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10481630265712738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720218930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289288103580475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720234754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132540762424469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720234755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720234755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720278658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814329862594604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720278658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720278658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720294246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542774945497513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720309787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502569377422333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720325640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023498922586441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720341445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788467526435852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720357016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428573191165924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720372155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826651215553284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720387553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09789703041315079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720387553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720387554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720432158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815671563148499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720432159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720432159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720447151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324009507894516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720462837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09697344154119492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720478291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09772045910358429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720493580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082935631275177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720508978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205840319395065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720524309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052875965833664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720540045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350563168525696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720540045, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720540046, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720584015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819533348083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720584015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720584016, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784720599258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024938523769379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720614587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428037494421005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720629973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129426419734955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720645428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965394228696823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720661413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539788752794266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720677262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793767869472504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720693065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09144231677055359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720693066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720693066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720736684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818242907524109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720736685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720736686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784720752720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023976281285286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720768216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218344628810883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720783704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842398554086685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720799022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023109406232834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720814352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639545321464539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720830053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470446199178696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720845862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962939471006393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720845862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720845862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720889059, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823770642280579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720889059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720889060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784720905020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154165118932724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720920738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640987753868103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720936417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016315370798111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720952052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071199178695679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720967622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643474757671356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720983403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863149166107178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720999053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885184466838837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720999054, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720999054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721043142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826308608055115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721043142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721043142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721058882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331970244646072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721074819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421782732009888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721090365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217311978340149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721106449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415861010551453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721122273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062001883983612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721138022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020582839846611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721154064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961039364337921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721154065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721154065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721197870, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825408577919006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721197870, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721197870, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721213668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142808198928833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721229352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781680047512054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721245060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755473375320435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721260334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767115652561188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721276544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030198335647583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721292632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404408723115921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721308576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11879545450210571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721308576, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721308577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721352543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828908562660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721352543, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721352544, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721368396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146412193775177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721384211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888591408729553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721399955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376173049211502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721415985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440954566001892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721431867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12130096554756165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721447880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379151046276093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721463615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419744789600372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721463616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721463616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721506232, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782987654209137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721506233, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721506233, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721522006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161126047372818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721538161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135096549987793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721554249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356890201568604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721570417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020569503307343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721586661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974390059709549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721602841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11534513533115387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721618487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048781126737595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721618487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721618488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721661313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831225395202637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721661314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721661314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784721676838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557736992835999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721693459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446021497249603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721709826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065036058425903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721725770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160239577293396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721742150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151488646864891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721758111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316564679145813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721774353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733804643154144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721774354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721774354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721817101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783629834651947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721817102, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721817102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784721833057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577410250902176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721849271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609680950641632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721865163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042168736457825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721881614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127186045050621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721897681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411838233470917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721913991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152319610118866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721930328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985612124204636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721930329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721930329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721974270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831918597221375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721974270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721974271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784721990169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520175635814667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722006072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519154906272888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722022171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703061521053314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722037937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450810939073563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722054044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666705667972565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722069960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809028148651123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722085864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005546897649765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722085865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722085865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722128978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837228178977966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722128978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722128979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722144732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587616264820099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722160899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478836297988892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722176759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132969930768013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722192872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247602105140686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722208659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403748601675034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722224512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725639760494232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722240334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945013165473938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722240335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722240335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722283461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7842088341712952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722283461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722283462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722299488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283378303050995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722315582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156386062502861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722331856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131308436393738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722347785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143805831670761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722363683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042846143245697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722379744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927733778953552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722395559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525558888912201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722395613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722395614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722438222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839191555976868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722438222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722438223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722453914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083188116550446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722469427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008932888507843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722484589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084040462970734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722499923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09945324808359146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722515799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101647675037384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722531132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187111049890518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722546694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900698602199554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.28325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722546740, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722546740, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722590212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7841613292694092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722590213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722590213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784722605855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08406013250350952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252641280, "lr": 1.2849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722621356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473915934562683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2865833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722637373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877791047096252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253296640, "lr": 1.28825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722653067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594905704259872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722668753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483582854270935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253952000, "lr": 1.2915833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722684552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424450784921646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2932499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722700501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365892201662064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254607360, "lr": 1.2949166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722700546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722700546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722743898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840080857276917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722743899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722743899, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784722759255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216554999351501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722774528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316088795661926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255262720, "lr": 1.2982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722790563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074608787894249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722806631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874128341674805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255918080, "lr": 1.3015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722822715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078343540430069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.30325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722838772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09872718155384064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256573440, "lr": 1.3049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722854872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679888725280762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722854872, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722854873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722897974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784669816493988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722897975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722897975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784722913876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316096246242523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257228800, "lr": 1.30825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722930276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10199180990457535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722946520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064729392528534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257884160, "lr": 1.3115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722962688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840563476085663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3132499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722978886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562644898891449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258539520, "lr": 1.3149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722994994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651440173387527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723011191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584577918052673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259194880, "lr": 1.3182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723011192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723011192, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723053926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839418053627014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723053926, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723053927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784723070237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985317081212997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723086583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859616100788116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259850240, "lr": 1.3215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723102761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675647854804993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.32325e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed295346581.log b/recommendation_v4/rcp_logs/gbs_16384/seed295346581.log new file mode 100644 index 000000000..615d272f3 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed295346581.log @@ -0,0 +1,1388 @@ +:::MLLOG {"namespace": "", "time_ms": 1784700452147, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784700452147, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784700473175, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "seed", "value": 295346581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784700473177, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784700473178, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784700473178, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784701370258, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784701370259, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784701370572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784701498242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13957616686820984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701511732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13927045464515686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701525002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388877034187317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701538673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387902796268463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701552133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857632875442505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701565699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386290192604065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784701579149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389015018939972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701579150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701579150, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701622853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989592432975769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701622853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701622854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784701636415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13915544748306274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701649916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903583586215973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701663546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891249895095825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701677286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389027237892151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701690838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388002336025238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701704978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897597789764404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701719189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867418467998505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701719190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701719190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701761587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4991030693054199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701761587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701761588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784701775319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871900737285614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701789454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871459662914276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701803071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386757493019104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701817347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860046863555908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701831036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852231204509735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701845400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388411819934845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701859104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838902115821838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701859105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701859105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701901792, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4992532730102539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701901792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701901793, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784701915594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864558935165405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701929983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833904266357422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701943952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798651099205017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701958036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848334550857544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701972957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804680109024048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784701987065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378539502620697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702001539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792461156845093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702001540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702001540, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702044482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4994494915008545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702044482, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702044482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784702058080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745862245559692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702071864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808664679527283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702086130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377759873867035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702100124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739851117134094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702114050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365707814693451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702128077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729120790958405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702142244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365933120250702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702142244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702142245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702184583, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49968764185905457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702184583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702184584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784702198426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714709877967834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702212227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13680338859558105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702226354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13763514161109924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702240702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696303963661194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702254621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566283881664276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702268496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13589628040790558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702282573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774576783180237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702282574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702282574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702326506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49995237588882446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702326506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702326506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784702340873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697797060012817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702355188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594967126846313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702369046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362307369709015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702383028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354161649942398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702397205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551735877990723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702411763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350596845149994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702425842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536252081394196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702425843, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702425843, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702468658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003459453582764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702468658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702468658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784702482855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345982849597931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702497171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660603761672974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702512029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380530595779419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702526446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437619805335999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702541104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626176118850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702555415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506832718849182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702570022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685756921768188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702570023, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702570024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702614068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008941888809204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702614069, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702614069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784702628356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679492473602295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702642643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674396276474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702656983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508644700050354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702671351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307932436466217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784702685705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327751725912094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702699721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338176280260086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702713522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350933462381363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702713574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702713575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702756477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015209913253784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702756478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702756478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784702770564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509628176689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702784631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294676959514618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702799400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150933384895325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702813426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681040704250336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702827520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245896995067596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702841433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316336691379547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702855904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244032859802246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702855904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702855905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702899015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022422671318054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702899016, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702899016, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784702912866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192984461784363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702927512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343478262424469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702941326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811052799224854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702955321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978525459766388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702969525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375459611415863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702983961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12832079827785492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702997901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300479620695114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784702997902, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784702997902, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784703040902, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032747983932495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784703040903, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784703040903, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784703055735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453617691993713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703069529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308075070381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703083653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344427466392517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703097691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347837746143341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703112085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359872817993164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703126230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343545794487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703140734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692359626293182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703140734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703140735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703184326, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045849680900574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703184327, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703184327, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784703198846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426348567008972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703213554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053633272647858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703228045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399618864059448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703242954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14035740494728088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703257639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037024438381195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703272756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323278248310089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703287212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828232884407043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703287213, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703287213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703330715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058749914169312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703330716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703330716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784703345369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264919489622116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703359905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108479976654053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703374709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370495855808258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703389324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539549708366394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703404330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231725990772247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703418526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883611023426056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703432966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12801657617092133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703432967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703432967, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703476442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072599053382874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703476442, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703476442, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784703490618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012802600860596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703505008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477855920791626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703519629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.142243891954422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703533981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391552865505219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703549134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145312666893005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703563462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374894857406616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703577820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591666519641876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703577821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703577822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703622557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5088711977005005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703622558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703622558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784703636695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346304714679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703651000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632752001285553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703665690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579228520393372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703680375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322588801383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703695101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430361449718475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703709808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293227553367615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703724104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261570036411285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703724105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703724105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703768545, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5106270909309387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703768545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703768546, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784703782800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197611272335052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703797152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447172939777374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703811740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14167185127735138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703826666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335456222295761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703841257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341942995786667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703855966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220030069351196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703870862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332024335861206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703870862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703870863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703914468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5126577615737915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703914469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703914469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784703928827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851480185985565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703943396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298574090003967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703957904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288197934627533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703972474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996622920036316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784703987457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799403607845306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704002590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153943419456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704017470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611677289009094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704017471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704017471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704060880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5149666666984558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704060880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704060881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784704075585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368284374475479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704090000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627567887306213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704104606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145941495895386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704119045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130088210105896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704133672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154831528663635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704148641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516822457313538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704163601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278286159038544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704163602, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704163603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704206748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5176932215690613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704206750, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704206750, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784704221266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358933299779892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704236650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322949081659317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704252182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775517046451569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704266990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629800081253052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704281925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855420053005219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704296784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418352603912354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704312062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389394998550415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704312062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704312063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704355344, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5205807089805603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704355357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704355357, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784704370429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503681123256683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704385610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15163502097129822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704400860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526108860969543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704415564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329825520515442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704430499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341647058725357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704445181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332026571035385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704460047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649754226207733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704460047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704460048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704504128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.523981511592865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704504129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704504129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784704519018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606154918670654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704534017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528087735176086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704549384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363173604011536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704564017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335419863462448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704578599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320505619049072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704593121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292557567358017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704607464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12555238604545593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704607464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704607465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704651622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5280427932739258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704651622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704651623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784704665935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242147862911224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704680674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248448073863983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704695664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348724663257599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704710405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352178454399109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704725207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656961917877197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704739836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287117749452591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704754273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964081764221191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704754274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704754274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704798593, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5343165993690491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704798594, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704798594, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784704812854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253383338451385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704827191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692014873027802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704841605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317278027534485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704856047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320868730545044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704870803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692447006702423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704885737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322203278541565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704900122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409411907196045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704900122, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704900123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704945500, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5431707501411438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704945500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704945500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784704960728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921807169914246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704975438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047336041927338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784704989980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304897665977478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705004667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972179055213928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705019016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12768200039863586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705033729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12297208607196808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705048735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271069049835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705048736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705048736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705092507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.555158257484436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705092507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705092507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784705107257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353226602077484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705122021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287204504013062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705136298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860034942626953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705151318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318899273872375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705165834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733492255210876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705180176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129330575466156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705194682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651756405830383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705194683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705194683, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705239414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5695282220840454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705239415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705239415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784705253670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12865950167179108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705267815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697109580039978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705282383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204234838485718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705297093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13211140036582947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705311958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090316951274872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705326443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331961750984192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705341456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994639575481415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705341457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705341457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705385297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5853544473648071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705385297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705385298, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784705400005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916463613510132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705414596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907521426677704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705429454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12862977385520935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705444378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185931742191315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705459041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323195993900299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705473404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857605516910553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705488485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939102947711945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705488485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705488486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705531964, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.598621129989624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705531965, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705531965, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784705546706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270987093448639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705561796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971951067447662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705576128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12680824100971222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705591648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651732563972473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705606858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12619341909885406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705621649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300029158592224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705636569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085468113422394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705636570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705636571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705679775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6128823757171631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705679776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705679776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784705694222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282716542482376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705709072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607577443122864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705724240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289796233177185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705739427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047687709331512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705754626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318102777004242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705769467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758594751358032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705784870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777704000473022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705784871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705784871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705827300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6289750933647156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705827301, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705827301, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784705842183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12766043841838837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705857210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256454885005951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705872187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265164017677307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705886941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332460641860962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705901960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870945036411285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705917116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12394080311059952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705932339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12632769346237183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784705932339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705932340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705975616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6437177062034607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705975616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705975616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784705990374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12348443269729614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706006240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257573366165161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706021292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12492460012435913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706036275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12386658042669296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706051130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1225750744342804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706066276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919476836919785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706081377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186513841152191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706081377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706081378, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706124274, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6610406041145325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706124274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706124274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784706139896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1238168403506279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706155271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12060323357582092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706170683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341191828250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706185900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200990229845047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706200611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521220952272415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706215267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630280315876007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706229998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356912553310394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706229999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706229999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706272019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6857621669769287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706272019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706272020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784706286777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193329244852066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706301500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11525056511163712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706316369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081122606992722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706331465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823108792304993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706346813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717721074819565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706362298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603929102420807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706377345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767126828432083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706377345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706377346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706421059, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.722382128238678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706421060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706421060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784706435789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515598744153976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706451379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472255736589432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706466693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182146817445755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706481689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250481009483337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706496887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116039752960205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706511850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218788474798203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706526775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819882154464722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706526842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706526842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706570413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.743534505367279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706570413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706570414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784706585217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294214427471161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706600672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135341227054596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706615977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754946410655975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706631493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122632026672363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706646335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204834282398224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706662048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783528327941895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706676974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560405254364014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706676975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706676975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706720854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.75544273853302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706720855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706720856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784706735747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429207772016525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706750692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224336177110672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706765681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995312035083771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706780432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973592102527618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706795099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391714423894882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706809510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11804880201816559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706824765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340317130088806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706824765, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706824766, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706868076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619572281837463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706868077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706868077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784706882891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639315634965897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706898147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120193749666214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706912897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545673757791519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706927958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085255652666092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706942964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1149802878499031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706957981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794450342655182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706972858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620200634002686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784706972858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784706972859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784707016877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7647040486335754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784707016878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784707016878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784707031617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880229622125626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707046475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818974673748016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707061111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384358257055283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707076036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092352494597435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707091258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127295047044754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707106404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305044054985046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707121697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156109720468521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707121698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707121698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707164043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664219737052917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707164044, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707164044, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784707179129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069960743188858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707193978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086893305182457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707209436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11946246027946472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707224751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12302196770906448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707239771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624555289745331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707255003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079861223697662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707270127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565650999546051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707270128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707270128, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707312430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679110169410706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707312431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707312431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784707327371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523889005184174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707342623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795799642801285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707358087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175260990858078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707373289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070951819419861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707388887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403083056211472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707404000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381706595420837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707419632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421766340732574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707419632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707419633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707463189, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688267230987549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707463189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707463189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784707478292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11820750683546066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707493552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677085816860199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707508933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448351502418518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707524136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397172510623932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707539352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776423454284668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707554764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660265386104584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707570515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395548284053802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707570515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707570516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707614094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695017457008362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707614095, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707614095, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784707629386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479946970939636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707645139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712074488401413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707660281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792982578277588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707675958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133309155702591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707691366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713309049606323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707706254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314524173736572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707721134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1166490763425827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707721135, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707721135, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707764964, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697186470031738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707764965, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707764965, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784707779879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998595505952835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707794759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428645998239517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707809450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728143155574799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707824095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474465042352676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707839439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182788968086243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707854720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09593044966459274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707870307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593542456626892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707870307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707870308, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707913030, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705028057098389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707913031, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707913031, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784707928068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517742484807968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707942831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959457606077194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707958638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11871393769979477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707973943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087842583656311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707988827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609767585992813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708003733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039588809013367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708018792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107060007750988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708018793, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708018793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708061664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709845900535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708061665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708061665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784708076709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460880398750305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708091444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496154427528381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708106469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916922986507416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708121706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710449516773224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708136982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586023330688477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708152378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162842810153961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708167970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929155349731445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708167971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708167971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708211296, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716566920280457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708211296, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708211297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784708226334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484659671783447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708241668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267823934555054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708256842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102062538266182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708272154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084715723991394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708287301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048530787229538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708302489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100511834025383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708317771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438425838947296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708317771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708317772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708360785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719554901123047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708360785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708360786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784708375971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434667766094208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708391308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978929162025452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708406402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446156561374664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708421830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104075163602829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708437072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063050776720047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708452510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033535823225975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708467908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982035100460052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708467908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708467909, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708510908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723216414451599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708510908, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708510909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784708525588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964022040367126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708541185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032966822385788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708556334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679009556770325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708571378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369756817817688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708586374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102259062230587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708601432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732024908065796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708616744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915398597717285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708616744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708616745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708660324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772561252117157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708660325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708660325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784708675279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564585030078888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708690331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292419791221619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708705340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844852030277252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708720745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611052811145782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708736147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004140228033066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708751588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761827230453491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708766806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108442023396492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708766807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708766807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708810868, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729083299636841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708810869, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708810869, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784708825746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666005313396454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708841557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103051364421844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708856518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438816249370575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708871708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148504376411438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708886726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798054933547974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708901703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312831193208694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708916692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108194500207901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708916692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708916693, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708961063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730993628501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708961063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708961063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784708975851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473780333995819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708990661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110733263194561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709005459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725847631692886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709020551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001984238624573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709035779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183114349842072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709051064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243492364883423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709066752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478958487510681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709066752, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709066753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709111464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773082435131073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709111465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709111465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784709126606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810476541519165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709141349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996126592159271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709157299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161646917462349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709172958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116140216588974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709188337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433209478855133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709203832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406321287155151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709219402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069929450750351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709219403, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709219403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709264199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730380296707153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709264200, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709264200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784709279703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081506311893463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709294910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385469138622284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709310479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007240414619446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709326335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051034182310104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709342174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452402174472809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709357780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219951510429382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709373745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317367851734161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709373745, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709373745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709417188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733471989631653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709417189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709417189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784709432435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920851677656174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709448309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707633197307587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709463566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673075914382935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709479259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478951573371887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709494370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11833846569061279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709509379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954742133617401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709524744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561432480812073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709524745, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709524745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709569334, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735484838485718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709569335, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709569335, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784709584386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344634741544724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709599643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366361379623413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709614919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554424464702606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709630308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366993188858032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709645708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932931303977966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709660994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898171365261078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709676081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217661947011948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709676082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709676082, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709721314, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736967206001282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709721314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709721315, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784709736767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868397355079651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709752335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477279871702194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709767340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459859669208527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709782614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038246899843216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709797628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065840944647789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709813069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313042998313904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709828104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846316814422607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709828105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709828105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709870904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773931086063385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709870904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709870905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784709885906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948474705219269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709900874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239019989967346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709916533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780103504657745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709932026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356497764587402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709947572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209286212921143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709963329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567861050367355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709978479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968734532594681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709978479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784709978480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784710022999, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739701867103577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784710023000, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784710023000, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784710037794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817234218120575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710053569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046759784221649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710068733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646992176771164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710083788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588032752275467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710098765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746665298938751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710113422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302180796861649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710128585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770076513290405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710128585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710128586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710172180, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742652893066406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710172181, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710172181, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784710187158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796566307544708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710201963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559845715761185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710217152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827981680631638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710232239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325493663549423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710247446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875147581100464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710262810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962513089179993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710278366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451638698577881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710278367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710278367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710322268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743571400642395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710322268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710322269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784710337709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791508853435516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710352560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210170596837997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710367927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123097687959671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710382767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028391569852829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710398259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010194778442383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710413243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230256617069244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710428011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578222572803497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710428011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710428012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710472348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744569182395935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710472348, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710472349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784710487019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030750647187233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710502238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028361529111862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710517513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081901490688324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710532670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994838178157806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710547432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472879558801651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710562541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085418164730072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710577679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925007611513138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710577680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710577680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710622117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745069265365601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710622118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710622118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784710637367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164618283510208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710652786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460987687110901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710667964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404352843761444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710682815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060253232717514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710698197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062375664710999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710713250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717925429344177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710728747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460489243268967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710728747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710728748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710772349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746050357818604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710772349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710772350, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784710787350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673008114099503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710802434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802753269672394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710817665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10030718147754669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710832890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959891974925995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710848312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698538482189178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710863642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196927934885025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710878556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298049449920654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710878557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710878557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710922576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747551798820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710922577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710922577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784710937892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223405063152313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710953185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850746929645538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710968389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898345708847046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710983945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112113296985626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710999224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079784482717514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711014038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963107645511627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711029585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198260843753815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711029586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711029586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711073818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749716639518738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711073819, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711073819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784711088805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931524634361267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711104174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404764115810394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711119152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067451685667038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711134319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151644587516785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711149247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080920472741127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711164623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236126720905304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711179689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401678830385208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711179690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711179690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711223712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747899889945984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711223713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711223713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784711239024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667860507965088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711254254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09822062402963638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711269391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049732565879822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711284778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395275056362152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711300458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11161624640226364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711316238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257307231426239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711332010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037968099117279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711332011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711332011, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711377371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751120924949646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711377371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711377371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784711392545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269594937562943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711407589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696907341480255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711423413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266377031803131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711438752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085072159767151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711454435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993054509162903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711469752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11869849264621735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711485084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179451271891594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711485085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711485085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711529987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751756906509399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711529988, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711529988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784711545420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780475080013275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711561184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997919738292694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711576416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505343019962311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711591774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250555515289307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711607343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115058034658432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711623254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659406870603561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711638988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843934118747711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711638988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711638989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711682338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752023339271545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711682338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711682338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784711697994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136235296726227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711713638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465019941329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711728927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203014105558395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711744905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216053366661072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711760647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010628938674927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711776347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774393379688263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711791980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030219495296478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711791981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711791981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711835435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752821445465088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711835435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711835435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784711850706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322090774774551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711866222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491353273391724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711881954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818074643611908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711897355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967309772968292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711912911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531258374452591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711928898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190153658390045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711944450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567486822605133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711944451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784711944451, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784711988889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775458037853241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784711988890, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784711988890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784712005016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618630588054657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712020575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461242288351059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712035818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656659841537476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712051612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409681499004364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712066873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149030387401581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712082081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663650929927826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712097103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604012757539749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712097103, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712097104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712139566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758594155311584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712139566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712139566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784712154386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059018224477768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712169426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761161148548126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712184412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190902650356293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712199372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432031750679016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712214390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439032316207886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712229296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09793457388877869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712244621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541936755180359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712244622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712244622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712287234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757909893989563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712287235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712287235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784712302347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596324503421783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712317581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990203380584717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712333080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0942482054233551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712348235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991670399904251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712363163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125955879688263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712378571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532797873020172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712393592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000612378120422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712393593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712393593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712436317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758732438087463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712436317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712436318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784712451751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609817504882812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712466797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300657153129578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712481679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155489832162857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712496552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769146144390106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712511690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732591152191162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712526687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039753407239914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712541774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083383321762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712541775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712541775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712584101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760679721832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712584102, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712584102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784712599011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002179443836212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712614095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570557415485382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712629543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625478625297546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712644998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737244039773941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712660660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271624475717545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712675961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333675146102905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712691151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567821562290192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712691151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712691151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712734756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776301383972168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712734756, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712734756, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784712749692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021130383014679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712765499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609882324934006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712780770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277725756168365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712796084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160688102245331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712811147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682842135429382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712826196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102628692984581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712841538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128888949751854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712841538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712841539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712883499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766081690788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712883499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712883500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784712898769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274626314640045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712913844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283750295639038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712928770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031703114509583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712943649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308007895946503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712958896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603135824203491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712974408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969556868076324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712989743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886236488819122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712989744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784712989744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784713033151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767153978347778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784713033151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784713033152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784713048804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012096032500267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713064250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09878285229206085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713079378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10067147761583328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713094178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812184423208237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713110211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630433261394501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713125539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105019211769104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713141006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723890364170074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713141007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713141007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713184007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776864230632782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713184008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713184008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784713199232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589642822742462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713214387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916868805885315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713229432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483500361442566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713244700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973035752773285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713259761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967764467000961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713275088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038578599691391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713290112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480546951293945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713290112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713290113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713333252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772340178489685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713333253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713333253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784713348645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09862935543060303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713364127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726273059844971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713379560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290852189064026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713395406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027908325195312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713411161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800963640213013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713426580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177048832178116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713441758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034964993596077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713441759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713441760, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713486320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774514555931091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713486321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713486321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784713502022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011776253581047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713517247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954532027244568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713532965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112223774194717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713548077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495088994503021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713563181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002858936786652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713578400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143483966588974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713593500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09970352053642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713593501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713593502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713637247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777019739151001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713637248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713637248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784713652524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504929721355438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713667575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204079747200012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713682409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697085410356522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713697548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398974269628525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713712775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700753331184387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713728129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313782632350922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713743618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214406251907349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713743618, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713743619, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713787083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780258655548096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713787084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713787084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784713802608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632907599210739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713818226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075646638870239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713833485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028570681810379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713848815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620227456092834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713864584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766138136386871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713880480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813899338245392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713896451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152288019657135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713896451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713896452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713940349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778012752532959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713940350, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713940350, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784713956066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577422380447388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713971720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818919539451599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713987464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124553233385086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714003315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495541781187057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714019193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871459543704987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714035125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615584254264832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714050892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670152306556702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714050892, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714050893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714094485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784315347671509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714094486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714094486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784714110233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128683388233185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714126256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105357110500336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714142256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075497418642044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714158891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209480464458466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714174848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924965143203735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714190516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430509388446808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714206254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320131480693817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714206254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714206255, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714250495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785643935203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714250495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714250496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784714266159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802292823791504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714282226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142293363809586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714297669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654892772436142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714313118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282733827829361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714328447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397533118724823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714344238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864043980836868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714359918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300921440124512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714359918, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714359919, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714403991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791324853897095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714403992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714403992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784714419504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447970569133759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714435179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909083485603333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714450915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283069849014282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714466699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109427958726883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714482765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698486119508743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714498557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476842522621155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714513933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101936548948288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714513934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714513934, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714559275, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794379591941833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714559276, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714559276, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784714574867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032030194997787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714590120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651960223913193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714605824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336887836456299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714621184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597541183233261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714636388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425235331058502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714651527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166437715291977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714666814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826329350471497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714666815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714666816, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714709837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796248197555542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714709838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714709838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784714725277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002817153930664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714740885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422342270612717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714756172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056360065937042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714771295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097478486597538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714786633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135414451360703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714801891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140049457550049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714817410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446792840957642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714817410, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714817411, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714861404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796503901481628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714861404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714861404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784714877046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079564169049263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714892438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102310411632061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714907747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09276972711086273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714922981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403728485107422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714938603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182388126850128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714953840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019408106803894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714969391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545764863491058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784714969391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784714969392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784715012817, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800098061561584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784715012817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784715012817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784715028148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09756751358509064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715043544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536845028400421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715058614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085633933544159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715073995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863869100809097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715089465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210210084915161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715104773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023004949092865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715119991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400596261024475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715119991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715119992, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715164137, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780346691608429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715164138, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715164138, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784715179184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249769687652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715194770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10884077101945877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715210165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965743213891983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715225280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317334532737732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715241066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269191116094589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715256921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09829481691122055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715272499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600118339061737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715272500, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715272501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715315524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808949947357178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715315525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715315525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784715331082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016143411397934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715346332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476227849721909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715362212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582304000854492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715377571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003211349248886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715393071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543353855609894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715408434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790842771530151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715423384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273230820894241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715423385, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715423385, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715467102, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808226943016052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715467103, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715467103, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784715482212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352812707424164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715497420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097678005695343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715512865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142894834280014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715528168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09868713468313217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715543593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474147647619247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715558848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287657916545868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715574406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075506567955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715574406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715574407, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715617922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809662222862244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715617923, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715617923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784715633458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573391616344452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715648893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444199293851852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715664659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225283354520798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715680338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785022377967834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715695560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405472666025162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715710737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09791626036167145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715726317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805934876203537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715726317, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715726318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715770807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811769247055054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715770807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715770808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784715785769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341829061508179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715801407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09687105566263199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715816597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09773723036050797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715831870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799866914749146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715847250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168245434761047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715862553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505443811416626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715878222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131686121225357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715878223, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715878223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715921288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816923260688782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715921289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715921289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784715936539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018398404121399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715952094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372035205364227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715967617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097894817590714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715982806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914048552513123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784715998438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458207130432129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716013811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835739225149155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716029473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09103193879127502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716029474, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716029474, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716072807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818500995635986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716072808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716072808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784716088899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249608755111694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716104420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010972261428833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716120173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793283581733704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716135840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021257936954498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716151826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591515898704529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716167475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468196868896484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716183259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903691709041595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716183260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716183260, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716227203, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819221019744873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716227204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716227204, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784716242969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523161828517914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716258693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619346052408218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716274278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157469660043716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716289791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025890707969666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716305447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672996193170547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716321235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082482561469078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716336838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903305023908615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716336839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716336839, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716379875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818952798843384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716379875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716379876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784716395655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327756941318512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716411530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365606844425201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716427021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230477154254913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716442882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398493707180023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716458325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610069334506989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716474062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018802598118782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716489941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939688980579376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716489942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716489942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716532952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7817438244819641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716532953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716532953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784716548831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399243772029877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716564602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808024555444717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716580333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715091973543167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716595527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702367126941681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716611939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295712947845459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716628108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412858426570892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716644293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11826032400131226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716644294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716644295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716686878, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78209388256073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716686879, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716686879, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784716702758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450108885765076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716718787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832184553146362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716734644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416899412870407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716750850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442917793989182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716767013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12015669792890549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716783226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344447731971741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716799275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417818069458008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716799276, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716799277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716843256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820115685462952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716843257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716843257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784716859158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160070151090622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716875602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353110522031784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716891748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229313910007477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716907964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014613509178162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716924282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993581265211105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716940568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504900455474854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716956504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106758862733841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784716956504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784716956505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784717000886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820450067520142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784717000887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784717000887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784717016713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542415618896484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717033154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424928903579712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717049292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058169603347778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717065185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175692826509476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717081313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506173014640808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717097064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11321870982646942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717113073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754293739795685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717113073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717113074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717157206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829115390777588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717157207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717157207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784717172938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521229147911072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717189003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600616574287415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717204506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014876514673233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717220831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125507801771164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717236611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433758586645126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717252641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538734287023544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717268874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960458964109421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717268874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717268875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717311937, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805144190788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717311938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717311938, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784717327571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150488406419754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717343386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477524787187576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717359887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731863230466843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717375976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531021654605865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717392443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066674143075943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717408495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795517265796661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717424527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112558841705322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717424528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717424528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717468206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825460433959961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717468207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717468207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784717484172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579445958137512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717500455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413683205842972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717516405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334630846977234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717532791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121726706624031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717548809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444684326648712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717564869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695438086986542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717581087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873205959796906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717581087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717581088, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717623901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783027708530426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717623901, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717623902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784717640188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269940435886383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717656596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546231806278229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717672857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10124969482421875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717688705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925869643688202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717704580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384128987789154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717720962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944317281246185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717736873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047864705324173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717736922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717736922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717781036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825866937637329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717781037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717781037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784717796774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005425751209259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717812350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118546336889267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717827466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084110498428345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717842807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09930329024791718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717858769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094643384218216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717874000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166378319263458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717889688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966642946004868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.28325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717889736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717889737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717933603, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782227098941803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717933603, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717933604, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784717949061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08273527026176453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252641280, "lr": 1.2849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717964498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501809418201447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2865833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717980565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934704542160034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253296640, "lr": 1.28825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784717996524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578688770532608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718012372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452081799507141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253952000, "lr": 1.2915833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718028265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457257926464081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2932499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718044249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367301106452942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254607360, "lr": 1.2949166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718044295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718044296, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718088783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826183438301086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718088784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718088784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784718104084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226867347955704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718119367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029936820268631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255262720, "lr": 1.2982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718135702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811120271682739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718151413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833308100700378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255918080, "lr": 1.3015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718167184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073065996170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.30325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718182833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855342656373978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256573440, "lr": 1.3049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718198279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066572442650795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718198279, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718198279, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718242581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819464206695557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718242582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718242582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784718258162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296022146940231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257228800, "lr": 1.30825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718274061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190264880657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718289591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654173046350479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257884160, "lr": 1.3115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718305029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834093391895294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3132499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718320666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573046654462814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258539520, "lr": 1.3149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718336135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643549263477325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718351895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566719621419907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259194880, "lr": 1.3182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718351896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718351896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718395495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822108268737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718395495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718395495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784718411478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963568091392517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718427558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777470469474792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259850240, "lr": 1.3215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718443415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602708160877228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.32325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718459425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097845509648323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260505600, "lr": 1.3249166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718475296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197171568870544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718491167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104045271873474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261160960, "lr": 1.32825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718507347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802723467350006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784718507348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718507348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718551885, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829669117927551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718551886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718551886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784718567295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598933696746826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261816320, "lr": 1.3315833333333332e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed313553746.log b/recommendation_v4/rcp_logs/gbs_16384/seed313553746.log new file mode 100644 index 000000000..60a6a1a65 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed313553746.log @@ -0,0 +1,672 @@ +:::MLLOG {"namespace": "", "time_ms": 1784922165375, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784922165375, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784922185840, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "seed", "value": 313553746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784922185842, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784922185843, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784922185852, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784922185852, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784923110364, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784923110365, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784923110661, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784923286495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14034254848957062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923300166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13995778560638428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923313751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391441524028778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923327465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388830542564392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923341103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881923258304596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923354726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892051577568054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784923368201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13931384682655334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923368202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923368202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923451952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49895021319389343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923451953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923451954, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784923465376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970088958740234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923478899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13943643867969513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923492607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13930463790893555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923506407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139229416847229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923520243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904637098312378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923534262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395665407180786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923548556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886556029319763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923548557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923548557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923591985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49912527203559875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923591985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923591985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784923606034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903075456619263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923620258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392870545387268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923633874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13926281034946442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923648345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391051709651947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923662277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390806883573532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923676783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391146332025528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923690561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389055699110031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923690561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923690561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923734440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49933359026908875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923734441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923734441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784923748360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883185386657715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923762951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883084058761597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923777106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386522650718689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923791497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386401653289795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923806299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842859864234924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923820306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382438987493515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923834936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384453922510147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923834937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923834937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923878386, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49958881735801697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923878387, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923878387, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784923892330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820911943912506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923906577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820016384124756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923921104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824273645877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923934965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138040691614151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923948865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730236887931824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923962890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378091275691986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923977329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726678490638733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784923977330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784923977330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784924020486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.499907910823822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784924020487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784924020487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784924034466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376279890537262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924048316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725055754184723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924062539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379377245903015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924076958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731792569160461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924090799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614974915981293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924104553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647881150245667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924118489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761216402053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924118490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924118490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924161782, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002878308296204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924161783, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924161783, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784924176113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731719553470612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924190443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365429162979126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924204701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365707516670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924219012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577911257743835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924233457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585197925567627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924248282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539224863052368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924262291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565492630004883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924262292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924262292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924304838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007304549217224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924304839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924304839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784924318887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13491088151931763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924333135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367110013961792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924348115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804218173027039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924362559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464275002479553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924377187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635891675949097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924391466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352977752685547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924406298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675731420516968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924406299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924406300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924449939, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013273358345032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924449940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924449940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784924464142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666415214538574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924478380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367979794740677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924492770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352170705795288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924507218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131240114569664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924521664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331607699394226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924535704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399943709373474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924549658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352946162223816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924549707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924549707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924593666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501997172832489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924593666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924593667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784924607818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514569401741028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924621979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990185618400574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924636910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13186797499656677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924650988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136742502450943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924665102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259582221508026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924679084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188402354717255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924693693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277778029441833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924693693, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924693694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924737435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027119517326355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924737436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924737436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784924751385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188621401786804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924766146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342782974243164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924779911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282966285943985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924793764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302568018436432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924807830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338598132133484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924822289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12860895693302155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924836335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300169974565506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924836335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924836336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924880918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037481784820557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924880919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924880919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784924895845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460445404052734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924909937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298560678958893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924924389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451936841011047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924938813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502360880374908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924953661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603031635284424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924968185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355836272239685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924983029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693177700042725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924983030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784924983030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925027079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052387118339539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925027080, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925027080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925041693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436082005500793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925056403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080790638923645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925070991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133977472782135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925085813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14022064208984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925100536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045787811279297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925115576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258662819862366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925130149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813424110412598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925130149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925130149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925175066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5068572759628296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925175066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925175066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925189790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536771595478058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925204254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313553750514984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925219055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710656762123108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925233742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536225259304047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925249287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323932409286499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925263641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289791613817215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925278270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12818694114685059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925278271, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925278271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925321633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5084006190299988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925321634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925321634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925336056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026157021522522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925350620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349189132452011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925365486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223510026931763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925380102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415119051933289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925395230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157391548156738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925409479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366323709487915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925424019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610854744911194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925424019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925424020, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925466800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.510086178779602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925466801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925466801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925481080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473208248615265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925495579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638761639595032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925510249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559499382972717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925524903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332017183303833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925539681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443133234977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925554656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303156197071075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925569322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243705034255981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925569322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925569323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925612471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118792653083801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925612472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925612472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784925627090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199833035469055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925641634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346013844013214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925656392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14185941219329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925671398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358628749847412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925686314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441112637519836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925701460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132329061627388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925716076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331784874200821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925716076, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925716077, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925758496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5139707922935486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925758497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925758497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784925773026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844731450080872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925787511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297364115715027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925801691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885211408138275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925816061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300225406885147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925830649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795724511146545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925845486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175609707832336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925860247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621962070465088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925860247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925860248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925904075, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5163049697875977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925904076, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925904076, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784925918892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684813678264618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925933507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647854328155518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925948240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141322135925293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925962822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009263575077057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925977570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13164228200912476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925992593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524477183818817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926007687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328985095024109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926007688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926007688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926051469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5189565420150757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926051470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926051470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926066275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590967655181885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926081563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234004378318787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926096530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787519931793213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926111335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363523304462433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926126252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385384351015091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926141144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344764232635498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926156362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910910487174988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926156363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926156363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926199436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5217757821083069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926199437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926199437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926214408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349618285894394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926229444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14775991439819336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926244777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516123592853546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926259587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299715518951416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926274807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411769270896912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926289745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331273317337036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926304696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366271823644638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926304696, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926304697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926348447, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5247534513473511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926348447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926348447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926363530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618093729019165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926378739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352235972881317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926394186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377463817596436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926408718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364359736442566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926423113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331673562526703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926437558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12910330295562744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926451820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12563253939151764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926451821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926451821, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926496316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5282393097877502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926496316, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926496316, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926510675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13274791836738586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926525353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250254094600677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926540163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504400849342346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926554872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538005948066711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926569825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367051750421524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926584879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885767221450806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926599878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998144328594208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926599879, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926599879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926643414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5335071682929993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926643414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926643414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926658189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266557455062866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926672923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723980844020844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926687597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330388069152832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926702229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212910294532776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926717201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711921334266663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926732292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227391242980957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926746754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451914489269257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926746754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926746754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926791322, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5404473543167114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926791323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926791323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926806598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945058941841125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926821294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305559277534485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926835801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070455193519592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926850307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000120222568512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926864565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280267834663391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926878943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341861426830292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926893687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740525603294373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926893688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926893688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926936802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.549720287322998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926936803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926936803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926951476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360294222831726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926966244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339826464653015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926980477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392356902360916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926995752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378536701202393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927010280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799063324928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927024828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300361007452011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927039633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718912959098816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927039634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927039634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927084179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5618278980255127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927084179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927084179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927098457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939715385437012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927112808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770360708236694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927127553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276824355125427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927142424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329708695411682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927157480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316293329000473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927172314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387885689735413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927187659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072721660137177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927187660, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927187660, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927230713, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5784400701522827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927230713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927230713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927245404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987112998962402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927259979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300901174545288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927274670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034231960773468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927289423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328262984752655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927303935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13407903909683228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927318186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005965948104858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927333147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098470866680145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927333148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927333148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927376240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5951531529426575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927376240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927376240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927390903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871117889881134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927405982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314857006072998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927420384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292611062526703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927435877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829825282096863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927451072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283256709575653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927466234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324734389781952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927481482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308638334274292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927481483, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927481483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927524683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6155332922935486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927524684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927524684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927539264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089345395565033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927554333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879741191864014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927569714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104639947414398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927585055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229236006736755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927600402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336984634399414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927615392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010074198246002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927630936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934017181396484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927630936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927630937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927674364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.636498749256134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927674365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927674365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927689186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040310144424438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927704008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12836918234825134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927718905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001194596290588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927733659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572485744953156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927748541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312469094991684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927763598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12680183351039886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927778851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13066783547401428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927778851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927778852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927821778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6608150601387024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927821778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927821779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927836535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723007798194885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927852160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12868568301200867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927867286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12825311720371246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927882151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830281257629395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927896952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12603062391281128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927911841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236049085855484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927926708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12471947073936462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927926709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927926709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927968937, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6877496838569641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927968938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927968938, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927984403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765946984291077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927999746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12375127524137497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928015060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174536943435669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928030458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234794408082962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928045486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978502571582794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928060122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12020723521709442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928074754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847937107086182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928074755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928074755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928117044, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7112540006637573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928117045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928117045, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784928131931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1232050359249115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928146462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884751915931702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928161267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539691686630249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928176217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12182827293872833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928191348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200210228562355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928206815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906507611274719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928221794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040332913398743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928221794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928221795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928266079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7377918362617493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928266079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928266080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784928280684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677883565425873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928296139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677147448062897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928311182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12041742354631424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928325999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449004709720612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928340964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423300206661224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928355640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449122428894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928370243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026551574468613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928370291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928370291, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928413801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7480444312095642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928413802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928413802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784928428376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544883251190186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928443699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309882253408432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928458985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11803142726421356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928474544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305390298366547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928489321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286622285842896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928504983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11972400546073914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928519584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769321024417877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928519584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928519585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928563158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7536473274230957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928563159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928563159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784928577797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151939183473587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928592551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331193149089813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928607556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139173805713654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928622327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008497327566147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928637082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046566367149353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928651655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711041629314423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928667083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355789005756378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928667084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928667084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928710714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.755949079990387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928710715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928710715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928725773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11574973165988922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928741131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294786632061005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928756078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11637299507856369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928771582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018119752407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928786755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387968063354492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928801867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732918232679367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928816852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672375559806824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928816852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928816853, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928861016, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592638731002808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928861017, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928861017, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928875800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085132360458374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928890721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083298996090889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928905267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331772804260254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928920139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863593220710754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928935369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282055079936981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928950511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203976929187775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928965743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135314404964447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928965744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928965744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784929009876, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7608283758163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784929009876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784929009877, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784929025015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696162283420563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929039882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901738703250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929055866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940069496631622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929071410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12111903727054596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929086677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355902999639511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929101999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053973436355591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929117193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639192700386047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929117194, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929117194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929161128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7626438736915588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929161129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929161129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784929176322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135602667927742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929191762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068248599767685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929207366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648652702569962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929222655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025096476078033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929238404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359985172748566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929253681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399298906326294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929269410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399416089057922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929269411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929269412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929314011, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7635855078697205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929314012, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929314012, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784929329430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878705769777298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929344803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670119524002075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929360183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518242418766022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929375496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346130073070526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929390692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771883070468903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929406071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621936619281769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929421908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301160603761673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929421908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929421909, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929465702, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651747465133667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929465703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929465703, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784929481185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428344994783401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929497005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678493231534958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929512112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852670669555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929527812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407694220542908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929542972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700219869613647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929557806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356762260198593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929572680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645941436290741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929572681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929572681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929616224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673921585083008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929616224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929616225, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784929631150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008653044700623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929646139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468670517206192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929660943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579819977283478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929675885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507984459400177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929691220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212881863117218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929706385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09604594111442566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929721956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579203814268112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929721956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929721957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929765040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678667306900024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929765041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929765041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784929780078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519352555274963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929794896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026601493358612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929810706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179376095533371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929826075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988186299800873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929841303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596532374620438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929856334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101854801177979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929871559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597026348114014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929871559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929871560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929914778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689653635025024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929914778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929914778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784929929937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542990267276764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929944939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517261922359467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929960257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969395190477371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929975583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743845254182816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929991036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604487359523773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930006494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116301566362381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930022278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914334654808044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930022278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930022279, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930065842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699422836303711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930065843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930065843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784930081006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491664707660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930096626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10199491679668427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930111984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804000496864319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930127269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832757502794266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930142406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508449375629425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930157461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103468596935272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930172675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491721332073212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930172675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930172676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930216334, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707123756408691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930216336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930216336, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784930231441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485590994358063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930246669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984967112541199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930261678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494448989629745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930277222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019408702850342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930292697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599091649055481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930308305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302960127592087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930323887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881300270557404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930323887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930323888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930368018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711547613143921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930368019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930368019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784930382946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997622668743134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930398709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297053307294846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930413814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704267024993896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930428919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318005084991455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930443944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240603983402252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930458804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692042112350464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930473929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893847793340683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930473929, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930473930, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930517509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718929052352905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930517510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930517510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784930532602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530628263950348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930547686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350555181503296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930562768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828660428524017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930578221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603472590446472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930593793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972472280263901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930609436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723234713077545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930624821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071121692657471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930624822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930624822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930668826, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728163003921509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930668827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930668828, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784930683576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672329366207123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930699246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015467494726181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930714706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422328114509583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930730253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191339999437332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930745546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852359235286713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930760758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327914893627167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930776132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076357364654541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930776133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930776134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930819969, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728846073150635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930819970, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930819970, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784930835136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362787544727325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930850344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090055108070374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930865551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763772577047348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930880966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995588660240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930896606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171594262123108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930912239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214211583137512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930928256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559244453907013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930928256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930928257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930971926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772803783416748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930971927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930971927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784930987467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107988640666008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931002548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934523701667786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931018737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590738594532013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931034661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500571668148041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931050220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469541490077972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931065716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404683649539948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931081332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061411052942276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931081332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931081333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931124520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732065320014954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931124520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931124520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784931140247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094638705253601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931155631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361470818519592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931171381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001426726579666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931187261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018174886703491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931203093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399547010660172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931218774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221614480018616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931234595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243182420730591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931234595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931234595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931278493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736842036247253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931278494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931278494, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784931293502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898794233798981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931309538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707272589206696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931325311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646525025367737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931341377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393536627292633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931356774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769388616085052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed462193368.log b/recommendation_v4/rcp_logs/gbs_16384/seed462193368.log new file mode 100644 index 000000000..4c9569a18 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed462193368.log @@ -0,0 +1,1380 @@ +:::MLLOG {"namespace": "", "time_ms": 1784719270658, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784719270658, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784719291040, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784719291041, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "seed", "value": 462193368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784719291042, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784719291091, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784719291092, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784719876257, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784719876258, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784719876555, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784720357924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893669843673706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720371866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896781206130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720385271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388426572084427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720399131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138778418302536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720412804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870219886302948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720426662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860449194908142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784720440126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908934593200684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720440127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720440127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720526866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4975312352180481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720526867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720526867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784720540471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892878592014313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720554134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388140767812729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720568027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861927390098572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720581861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386427879333496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720595716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866238296031952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720609983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871459662914276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720624398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808685541152954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720624399, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720624399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720667895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4977196753025055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720667896, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720667896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784720681810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851550221443176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720696162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847818970680237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720710122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385708600282669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720724374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383846402168274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720738050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840961456298828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720752444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392565220594406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720766486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824033737182617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720766487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720766487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720809273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4979620575904846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720809274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720809274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784720823417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853728771209717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720837823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830606639385223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720851652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778305053710938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720865724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828372955322266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720880447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787047564983368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720894449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777440786361694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720908886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773104548454285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720908887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720908887, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720951396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4982835054397583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720951396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720951396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784720964934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731138408184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720978717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378984898328781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720992917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771380484104156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721006863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373315155506134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721020991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364738792181015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721035088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734737038612366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721049239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364866942167282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721049240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721049240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721091998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4986475110054016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721091999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721091999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784721105619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718420267105103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721119136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669535517692566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721133085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13765046000480652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721147357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696137070655823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721161213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564667105674744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721175040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359473615884781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721189013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137473464012146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721189014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721189014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721232643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49907079339027405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721232643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721232644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784721247000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716834783554077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721260917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618701696395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721274656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617223501205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721288690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353398561477661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721302859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554197549819946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721317492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514447212219238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721331800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544785976409912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721331801, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721331801, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721374058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49958303570747375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721374059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721374059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784721388478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347178965806961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721403022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366688758134842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721417656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807165622711182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721431890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444191217422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721446306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624411821365356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721460470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516557216644287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721475060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367613971233368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721475061, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721475062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721518073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003060698509216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721518074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721518074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784721532598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657723367214203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721547127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367053985595703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721561753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350792646408081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721576340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310189962387085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784721590658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303568959236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721604714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403113186359406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721618614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351870596408844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721618665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721618665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721661495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5011168122291565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721661495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721661495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784721675621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350448727607727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721689585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130008727312088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721704682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181743025779724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721718736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678358495235443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721732884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265156745910645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721746970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188216090202332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721761520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270971179008484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721761520, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721761521, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721805148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019113421440125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721805149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721805149, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784721818996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203981518745422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721833597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437283039093018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721847326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12853457033634186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721861217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997624278068542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721875328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13385823369026184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721889696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12868638336658478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721903570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13014987111091614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721903571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721903571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721948285, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030502676963806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721948286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721948286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784721962816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344757378101349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721976792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308599591255188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721990990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448700308799744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722005066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503821194171906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722019637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359674334526062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722033868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352365791797638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722048336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690617680549622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722048337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722048337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722091257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046063661575317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722091258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722091258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784722105822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429683446884155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722120633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058871030807495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722135365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394570350646973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722150466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14039936661720276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722165185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055747747421265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722180080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249847292900085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722194536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813847303390503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722194537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722194537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722238212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061318278312683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722238213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722238213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784722252958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393048405647278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722267402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125941157341003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722282293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715136051177979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722296924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542446494102478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722311912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323881447315216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722326331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883734703063965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722340812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12807968258857727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722340812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722340813, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722383145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077716112136841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722383146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722383147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784722397573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011398911476135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722412095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480791449546814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722426951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227595925331116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722441480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387781381607056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722456065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131576806306839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722470340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375312089920044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722484878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604183495044708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722484878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722484879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722527446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096477270126343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722527447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722527447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784722541893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346476972103119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722556466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641908764839172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722571245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356702595949173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722585912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323161005973816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722600556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444699347019196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722615084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297362625598907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722629417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256192207336426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722629418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722629418, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722672534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5116674900054932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722672535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722672535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784722686741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202986121177673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722701232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435685634613037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722715853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141645610332489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722730941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347189128398895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722745778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422992825508118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722760484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324262022972107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722775005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331416666507721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722775006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722775006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722818420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5139864683151245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722818421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722818421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784722832767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871824741363525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722847287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315482437610626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722861585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12878021597862244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722876117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002726435661316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722890874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791847229003906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722905793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315857172012329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722920249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363862305879593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722920250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722920250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722963919, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5166791081428528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722963919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722963919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784722978492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674697279930115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722992696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646166026592255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723007224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147017359733582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723021542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300285905599594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723036037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131544291973114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723050856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530004024505615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723065671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265833258628845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723065672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723065672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723109393, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5197435021400452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723109394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723109394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784723123928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595983386039734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723139235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234946131706238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723154474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795632123947144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723169437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634918630123138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723184608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856303691864014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723199582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344960331916809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723214891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916543126106262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723214891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723214892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723258242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5228663086891174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723258242, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723258242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784723273414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497188687324524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723288579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15076710283756256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723303918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515877723693848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723318813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298112154006958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723333965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426166772842407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723348786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331261694431305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723363754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365644931793213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723363755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723363755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723406214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5266435146331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723406214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723406215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784723421285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609597086906433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723436376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135352224111557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723451901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377362489700317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723466742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355298340320587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723481565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332196593284607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723496435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294664442539215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723511202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256832629442215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723511202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723511215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723554249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5311552882194519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723554274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723554275, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784723569012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269229233264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723584064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245698809623718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723598906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349279135465622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723613940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523846864700317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723629403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669750094413757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723644514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288905143737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723659430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984086573123932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723659431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723659443, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723702642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5371183156967163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723702666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723702666, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784723717431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262006640434265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723732126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270495355129242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723746942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322393596172333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723761870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222065567970276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723777117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705254554748535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723792641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233700394630432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723807261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426288962364197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723807262, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723807262, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723851476, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5451311469078064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723851477, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723851477, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784723866458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294240951538086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723881067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304645836353302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723895596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064908981323242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723910290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297428011894226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723924523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12789687514305115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723939090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324057519435883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723954129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12725773453712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723954130, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723954131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723997248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5555940866470337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723997248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723997249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784724012097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358340084552765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724027037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333588808774948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724041453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916289806365967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724056701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345454633235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724071620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780356407165527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724086372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12967276573181152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724101435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692376971244812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724101436, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724101436, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724144684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5680530667304993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724144684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724144685, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784724159154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907150387763977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724173958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12757593393325806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724189103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253918290138245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724204343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266819715499878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724219737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147008419036865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724234711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362927734851837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724249833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060015439987183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724249834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724249834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724293407, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5810766220092773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724293407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724293408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784724308306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949863076210022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724323120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297198086977005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724337948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988050282001495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724352774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279208540916443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724367501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362397253513336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724382061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12953057885169983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724397348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044844567775726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724397349, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724397349, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724440477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5932912230491638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724440477, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724440477, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784724455381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12837481498718262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724470574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109003007411957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724485024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12858253717422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724500797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12794435024261475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724516254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747445702552795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724531582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191258907318115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724546970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327086240053177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724546971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724546971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724590388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6071704626083374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724590389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724590389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784724605126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987375259399414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724620279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12825949490070343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724635701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038456439971924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724651127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208064436912537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724666577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332387804985046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724681544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944380939006805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724697100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12901416420936584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724697101, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724697102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724740645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6233689188957214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724740646, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724740646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784724755474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970036268234253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724770446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278052031993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724785640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12927353382110596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724800662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537994027137756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724815865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060256838798523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724831281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607181072235107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724846870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294553279876709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724846870, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724846871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724891749, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6358329653739929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724891750, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724891750, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784724906773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12616634368896484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724922333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12818750739097595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724937362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717963755130768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724952151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721920013427734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724966882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12541484832763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724981664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246383726596832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724996467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382277846336365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724996468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724996468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784725039847, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.651576042175293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784725039847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784725039847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784725055274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12696433067321777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725070502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12304140627384186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725085609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165928766131401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725100599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12262792885303497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725115782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11833040416240692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725130604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940717697143555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725145623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700749397277832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725145623, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725145624, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725189348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6671977043151855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725189349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725189349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784725204456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12235084176063538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725219247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795192956924438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725233961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461281031370163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725248724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12096667289733887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725263931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193600594997406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725279245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181427612900734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725294193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924862325191498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725294193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725294194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725338153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7128434777259827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725338153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725338154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784725352848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604639887809753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725367939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600340902805328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725382653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969386041164398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725397346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478599905967712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725412115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331582814455032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725426708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416161805391312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725441551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969021916389465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725441597, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725441598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725484549, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7390022277832031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725484550, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725484550, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784725499183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466140300035477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725514520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127132847905159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725529627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11796179413795471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725545124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259748041629791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725559941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219245195388794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725575547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11931085586547852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725590448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730534046888351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725590448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725590449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725633873, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7554304599761963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725633873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725633874, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784725648721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500144004821777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725663400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113107830286026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725678431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079579591751099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725693179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985603928565979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725708094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425853729248047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725722787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695531755685806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725738287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277531087398529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725738287, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725738288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725781298, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7614619135856628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725781299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725781299, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784725796420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589442193508148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725811868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296522617340088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725826684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516264081001282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725841873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125316679477692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725856668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403702199459076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725871364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733340680599213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725886020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650570690631866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725886020, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725886021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725929682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7639240026473999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725929683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725929683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725944231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851176828145981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725958785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760509967803955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725973092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270982027053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725987762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858064889907837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726002784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191538721323013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726017629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194587707519531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726032594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031801253557205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726032595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726032596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726077720, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652116417884827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726077720, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726077720, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784726092578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651849210262299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726107171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842617601156235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726122683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920344829559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726138033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214369386434555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726153063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404015868902206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726168226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083677411079407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726183312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496612429618835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726183312, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726183313, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726227192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7666016221046448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726227193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726227193, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784726242014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458340287208557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726257237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774167627096176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726272672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747124791145325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726288062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016599833965302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726303714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381636559963226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726318976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403126269578934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726334429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440704226493835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726334429, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726334430, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726377076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677991390228271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726377077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726377077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784726392096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834365129470825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726407078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652053356170654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726422124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515066981315613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726437012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292680352926254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726451818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11846679449081421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726466749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726063698530197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726482147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430469155311584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726482147, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726482148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726525041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690052390098572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726525041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726525042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784726540034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494681984186172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726555537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713953524827957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726570266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853086411952972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726585600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336154490709305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726600548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107160285115242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726615263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399685800075531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726629976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167742982506752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726629977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726629977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726672443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696030735969543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726672444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726672444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784726687182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935831815004349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726702001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146104708313942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726716664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595807433128357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726731264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469216108322144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726746346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164809226989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726761346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09619437158107758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726776602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640332102775574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726776603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726776603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726819291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702373266220093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726819292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726819292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784726836184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473702847957611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726850759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028195917606354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726867496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799097061157227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726883438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970813035964966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726901224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574804246425629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726918763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090966522693634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726939534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649309307336807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726939535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726939535, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726981940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708130478858948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726981941, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726981941, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784727003769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527485609054565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727028385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580509901046753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727048700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933275520801544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727063847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678771883249283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727079013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611891746520996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727094163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158455908298492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727109572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092674732208252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727109573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727109573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727152788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714694738388062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727152788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727152788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784727167647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10437706857919693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727182885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235366225242615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727198069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832292586565018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727213341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899942368268967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727228350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047234982252121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727243455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086062550544739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727258575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460534691810608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727258575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727258576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727300567, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717311382293701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727300567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727300567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784727315471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445437580347061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727330558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021249949932098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727345317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490051656961441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727360521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008711904287338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727375614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552039742469788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727391020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266974568367004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727406432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964898020029068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727406433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727406447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727449840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723022103309631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727449866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727449866, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784727464769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027772188186646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727480406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338522493839264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727495709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640020668506622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727511095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035483330488205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727526324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232290625572205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727541526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633393377065659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727556786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897031426429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727556786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727556799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727600468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772521436214447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727600494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727600495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784727615708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571062564849854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727631013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418636351823807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727646010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774342715740204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727661517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584823787212372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727677124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924822092056274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727692801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750848054885864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727708433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075285077095032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727708434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727708447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727751463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730774879455566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727751489, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727751489, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784727766576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668856650590897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727781935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163629055023193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727797026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422785580158234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727812157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186198353767395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727827111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828734934329987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727841972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328025907278061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727856864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798776149749756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727856864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727856865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727900516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732545733451843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727900517, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727900517, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727915494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313259810209274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727930438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139428615570068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727945480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767336189746857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727960725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034355521202087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727976027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116548478603363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727991271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241618543863297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728006994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463912785053253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728006995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728006995, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728050870, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733749747276306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728050870, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728050870, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784728066104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780662298202515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728081046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984919428825378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728097188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11602994054555893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728113005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504407227039337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728128496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144031286239624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728144001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039624810218811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728159669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660772770643234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728159669, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728159670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728203750, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773219108581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728203751, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728203751, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784728219470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076157540082932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728234799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137712299823761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728250377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980388522148132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728266099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983070731163025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728281697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419390141963959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728297230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192235350608826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728313089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298248171806335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728313090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728313090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728356724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735974192619324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728356725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728356725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784728371863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093788668513298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728387499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712501406669617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728403223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723394900560379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728419156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428209394216537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728434585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844397336244583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728450172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095581203699112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728466033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558160930871964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728466034, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728466034, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728510431, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740015387535095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728510431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728510431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784728525853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359748244285583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728541404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344676464796066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728557091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11566121876239777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728572772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367519199848175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728588619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926980525255203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728604193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897397249937057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728619381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195182800292969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728619382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728619382, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728663727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741607427597046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728663727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728663727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784728679357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867741703987122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728694965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465399920940399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728709993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442653298377991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728725325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400320589542389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728740427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065400093793869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728756022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259495675563812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728771257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788720101118088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728771257, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728771258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728814370, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746317386627197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728814370, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728814371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728829734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916749387979507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728844782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245875269174576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728860381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772121697664261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728875948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330729931592941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728891648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202420502901077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728907517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548358410596848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728922907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969821572303772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728922907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728922908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728967826, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747117280960083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728967827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728967827, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728982813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788844525814056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728998415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101015955209732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729013723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629718750715256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729028856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617353022098541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729043830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698670148849487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729058594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353559255599976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729073597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073928028345108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729073597, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729073598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729117639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750006914138794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729117640, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729117640, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784729132662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772106051445007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729147534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052151769399643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729162672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849309712648392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729177766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293030738830566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729193146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899125039577484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729208531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936196148395538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729224111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529421269893646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729224111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729224112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729269106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751185297966003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729269107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729269107, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784729284549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761916637420654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729299506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171954333782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729314815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166489124298096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729329882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243584215641022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729345510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017955303192139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729360866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245376825332642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729376084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518445074558258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729376085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729376085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729420308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753562331199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729420308, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729420308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784729435388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327232629060745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729450830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043381154537201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729466394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823230445384979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729481628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094655692577362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729496656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471397638320923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729512058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771641135215759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729527458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915465652942657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729527458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729527459, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729571925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755560874938965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729571925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729571925, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784729587383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057352274656296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729603002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432210564613342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729618431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408469289541245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729633364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543924570083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729648866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980843961238861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729664030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731787979602814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729679715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442028939723969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729679716, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729679716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729723420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756752371788025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729723421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729723422, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784729738687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656488686800003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729753946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809750109910965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729769367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004767775535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729784822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903456807136536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729800309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09654764086008072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729815964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214927792549133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729831043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398636758327484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729831044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729831044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729874408, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758289575576782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729874408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729874409, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729889905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021241620182991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729905587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806003212928772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729921211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854384303092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729937167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031462252140045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729952915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790829360485077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729968094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877801477909088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729983676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014413982629776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729983677, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729983678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784730027229, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761046886444092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784730027229, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784730027229, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784730042319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986441373825073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730058014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407352447509766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730073232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669203102588654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730088440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145322024822235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730103427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806909948587418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730118769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233828961849213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730133945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425819456577301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730133945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730133946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730177553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760875225067139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730177554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730177554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784730192849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647537559270859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730208263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098326176404953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730223593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001647263765335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730238931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427667200565338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730254526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142121255397797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730270098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123659536242485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730286061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411910712718964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730286061, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730286062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730330518, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764241099357605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730330518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730330518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784730345931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268926620483398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730361028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063736081123352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730377167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200667381286621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730392832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045487225055695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730408709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964588820934296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730424359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185777336359024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730440013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11743141710758209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730440014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730440014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730484016, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766559720039368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730484016, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730484016, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784730499441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783412843942642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730515101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979273915290833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730530503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455871164798737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730546077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173561215400696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730561685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458539962768555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730577828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11688542366027832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730593633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838210582733154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730593633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730593634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730637491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767434120178223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730637491, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730637492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784730653261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091819405555725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730669285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443833470344543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730684703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119924858212471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730700699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189533770084381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730716637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967712849378586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730732784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711801052093506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730748696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099385991692543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730748696, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730748697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730792913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768189311027527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730792913, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730792913, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730808455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341968178749084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730824137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470253974199295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730840128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082114726305008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730856020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938552767038345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730871914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115289106965065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730888294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134612560272217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730904387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11574308574199677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730904387, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730904388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730947685, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777043342590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730947685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730947685, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730964062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603327095508575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730980203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454711854457855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730995715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673632264137268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731011761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038413941860199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731027431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145619511604309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731043001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611574351787567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731058535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058407723903656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731058535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731058536, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731103512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773395776748657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731103513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731103513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784731118784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958661139011383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731134221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770374536514282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731149680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126145929098129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731165237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472145676612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731180517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395751893520355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731195707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983285903930664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731211403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527652502059937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731211404, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731211404, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731255483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774782180786133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731255484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731255484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784731270865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588622093200684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731286549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952152520418167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731302252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09394059330224991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731317702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963355958461761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731332851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210288107395172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731348319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495726019144058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731363984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003443598747253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731363984, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731363985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731406920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778375744819641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731406920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731406920, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784731422762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558995604515076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731438260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028808206319809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731453668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477480828762054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731468984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09771712124347687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731484611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637752711772919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731500078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384131968021393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731515740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045450925827026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731515740, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731515741, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731558682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780590057373047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731558682, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731558682, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784731574151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988564252853394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731589641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553757846355438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731605573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593345761299133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731621323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713745653629303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731637314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246780514717102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731653090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314980149269104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731668761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500646382570267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731668761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731668762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731711707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783434987068176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731711709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731711709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784731726941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200902074575424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731742391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553476214408875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731757558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226352512836456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731772875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073602735996246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731788099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064261719584465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731803118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203736275434494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731818359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312112957239151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731818360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731818360, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731861104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788947224617004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731861105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731861105, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731876304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202586650848389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731891408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295695066452026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731906541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998462438583374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731921633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262124240398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731937108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562968254089355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731952664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979302227497101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731968143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899887442588806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731968143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731968148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784732010883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779147207736969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784732010884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784732010884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784732026900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109525918960571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732042648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853309392929077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732057936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031123459339142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732072858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09779283404350281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732090742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587599873542786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732106177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034076139330864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732121612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708893835544586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732121613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732121614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732164557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794066667556763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732164558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732164559, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784732179673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548287630081177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732194605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766482561826706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732209616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431067645549774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732227004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863699972629547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732241996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912759810686111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732257109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374850034713745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732272094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353760421276093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732272094, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732272096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732315478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796381711959839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732315480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732315480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784732330764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09808877110481262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732346304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627345740795135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732361697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275260359048843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732377318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955418646335602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732392805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757777839899063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732408110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022800207138062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732423142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248042643070221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732423142, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732423143, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732466546, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799597382545471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732466547, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732466548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784732482055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10116896033287048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732497234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965381562709808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732512977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053612291812897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732528333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044645607471466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732543693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934377670288086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732559086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965455323457718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732574343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953175485134125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732574344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732574344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732617628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7801960706710815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732617629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732617629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784732632978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049535870552063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732648397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175870358943939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732663666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629284381866455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732679111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320675373077393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732694687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654442012310028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732710315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273766309022903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732726088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018814742565155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732726089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732726089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732768452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780381977558136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732768453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732768453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732784333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607306659221649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732800285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110554099082947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732815963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235553234815598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732831558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638972371816635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732847276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708479583263397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732862796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779450088739395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732878427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136423796415329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732878428, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732878428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732921571, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807738184928894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732921572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732921572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732937193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456795990467072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732952678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840998589992523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732968225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110549196600914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732983902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367947608232498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732999612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084848940372467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733015344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535597801208496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733030943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610745847225189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733030944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733030945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733073994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809861302375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733073995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733073995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784733089851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073896288871765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733105661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055272817611694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733121519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965836048126221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733137733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174846440553665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733153639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840727388858795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784733169153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394806206226349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733184663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287575423717499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733184663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733184664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733228157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813759446144104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733228158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733228158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784733244045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765742510557175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733260455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114536225795746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733276192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581330955028534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733291833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130990982055664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733307362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113710418343544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733323170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828092694282532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733338806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294449865818024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733338806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733338806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733383184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820587754249573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733383185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733383185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784733398724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473743617534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733414593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835939645767212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733430528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286681890487671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733446426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069492995738983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733462564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683862864971161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733478458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448743402957916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733493875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069124937057495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733493876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733493876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733537048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820477485656738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733537049, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733537050, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784733552751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001949965953827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733567717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061709076166153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733583330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019812524318695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733598538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486561805009842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733613621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386441648006439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733628506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110839247703552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733643560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766088426113129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733643561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733643561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733686757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.781900942325592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733686758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733686758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733701998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990837424993515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733717357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372111201286316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733732606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070177167654037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733747869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766922891139984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733763289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110069811344147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733778731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156048834323883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733794193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405881702899933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733794193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733794194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733837830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782390832901001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733837830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733837830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733853532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790228098630905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733869142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149426758289337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733884555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09235085546970367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733899696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384237766265869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733915510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180173069238663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733930768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155151784420013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733946554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046365350484848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733946555, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733946555, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733990429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823275923728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733990430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733990430, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784734005920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0954965353012085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734021423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053522527217865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734036622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818882286548615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734052009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09888565540313721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734067567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147744417190552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734082833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184995085000992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734097948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384805500507355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734097948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734097949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734140854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825010418891907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734140855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734140855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784734155895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020069420337677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734171499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877172648906708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734186844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09560234844684601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734201967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304555296897888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734217708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216248035430908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734233474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982958972454071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734248903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568936169147491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734248903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734248904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734292192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7827907204627991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734292193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734292193, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784734307571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182933509349823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734322664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362203419208527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734338184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622374713420868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734353807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993339866399765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734369386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555823147296906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734384827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728155076503754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734400011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263123363256454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734400011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734400012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734442765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828752994537354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734442765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734442765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784734457991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375629365444183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734473425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100289434194565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734488861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115846991539001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734504170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09910757839679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734519546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462293028831482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734534626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273322999477386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734550008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095466673374176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734550008, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734550009, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734592370, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7827110886573792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734592371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734592371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734607760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541138052940369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734623168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464529693126678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734638883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208207368850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734654590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764418542385101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734669919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399505496025085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734684973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769628942012787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734700384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809520840644836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734700384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734700385, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734744671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783117413520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734744672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734744672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734759628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293124616146088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734775402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693505614995956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734790687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09725377708673477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734805950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792208462953568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734821275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158513486385345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734836524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487066209316254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734852175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285562813282013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734852176, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734852177, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734895728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834545373916626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734895729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734895729, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734910974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042782664299011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734926395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354583710432053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734941720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081326961517334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734956999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087978333234787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734972737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450433194637299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734988443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800415277481079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735004250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09056134521961212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735004250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735004251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735048753, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837228775024414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735048753, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735048753, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784735064694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273706912994385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735080227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10146333277225494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735095748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864144563674927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735111290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987287759780884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735126833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105409637093544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735142631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432331264019012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735158400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991472452878952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735158400, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735158401, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735202501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839759588241577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735202502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735202502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784735218121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505156755447388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735233721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613352060317993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735249327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126958787441254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735264954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961239039897919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735280448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631899535655975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735296079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863429307937622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735311407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09883023798465729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735311408, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735311408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735356462, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840574979782104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735356463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735356463, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784735371944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127198189496994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735387477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379203408956528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735402681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10215102136135101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735418433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417039692401886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735433846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600510239601135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735449388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207997262477875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735465244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938268899917603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735465245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735465245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735509525, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840730547904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735509526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735509526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784735525172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381416022777557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735540709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802052170038223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735556435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070374995470047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735571647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107216015458107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735587500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255666077136993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735603463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359667241573334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735619323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11737973988056183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735619324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735619324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735663529, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784687876701355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735663530, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735663530, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735679280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145341545343399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735694990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826212167739868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735710606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363540589809418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735726683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451896488666534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735742555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12105835974216461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735758554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337463557720184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735774238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337663978338242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735774239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735774239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735818458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7846050262451172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735818458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735818458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735834028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679750680923462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735849976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328455805778503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735865666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281918734312057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735881249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013953387737274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735897143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964155197143555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735913093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148909330368042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735928636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988613963127136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735928637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735928637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735973497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7846099734306335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735973498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735973498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735988948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556240916252136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736005115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456804722547531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736021243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038945615291595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736037025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122534424066544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736053140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474299430847168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736068675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328279972076416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736084520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727095395326614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736084521, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736084522, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736129505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7849793434143066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736129507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736129507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784736145132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550366878509521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736161277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159433126449585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736176998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070811003446579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736193323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286839842796326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736209417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150108054280281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736225358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620332300662994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736241533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923641920089722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736241534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736241534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736285830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784963846206665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736285831, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736285831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784736301555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467503011226654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736317508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470887809991837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736333637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740408301353455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736349430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500656068325043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736365685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630430281162262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736381631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819120705127716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736397299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098582297563553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736397300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736397302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736440292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7848413586616516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736440293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736440293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784736455953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611863434314728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736471987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369881987571716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736487800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318433284759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736503944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235027015209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736519839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458268225193024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736535846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671311616897583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736551763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943952202796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736551764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736551766, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736595419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852113246917725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736595420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736595420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736611629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290726810693741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736627941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447322368621826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736644143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123225301504135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736659910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093830019235611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736675678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336621850728989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736691780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887601971626282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736707659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500390082597733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736707733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736707733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736752221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7847932577133179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736752222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736752222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736768017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086378455162048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736783912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102714598178864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736799334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098101943731308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736815028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09872197359800339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736831150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046815872192383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736846890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241229087114334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736862918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939529538154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.28325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736862973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736862974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736906765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784920871257782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736906765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736906765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736922572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08410438895225525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252641280, "lr": 1.2849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736938033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469726473093033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2865833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736954086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900053381919861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253296640, "lr": 1.28825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736969897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11606618762016296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736985508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469963192939758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253952000, "lr": 1.2915833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737001165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104792021214962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2932499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737017201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305969417095184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254607360, "lr": 1.2949166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737017257, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737017257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737060839, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7856467962265015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737060840, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737060840, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784737076177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205554962158203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737091479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314513742923737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255262720, "lr": 1.2982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737107561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758110880851746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737123285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828036814928055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255918080, "lr": 1.3015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737139105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743861645460129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.30325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737154800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0990375280380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256573440, "lr": 1.3049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737170566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070919930934906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737170567, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737170567, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737213357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7854977250099182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737213357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737213358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784737228868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325668752193451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257228800, "lr": 1.30825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737244958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166488587856293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737260429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059676855802536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257884160, "lr": 1.3115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737276064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848291218280792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3132499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737291511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512186586856842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258539520, "lr": 1.3149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737306815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064056009054184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737322410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061730682849884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259194880, "lr": 1.3182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737322411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737322412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737365847, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852309942245483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737365847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737365848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784737381728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961703211069107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737397707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077723503112793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259850240, "lr": 1.3215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737413641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10624031722545624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.32325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737429659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056247353553772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260505600, "lr": 1.3249166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784737445435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212203115224838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265833333333332e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed530691108.log b/recommendation_v4/rcp_logs/gbs_16384/seed530691108.log new file mode 100644 index 000000000..956873cbc --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed530691108.log @@ -0,0 +1,669 @@ +:::MLLOG {"namespace": "", "time_ms": 1784931742455, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784931742455, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784931763370, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "seed", "value": 530691108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784931763372, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784931763373, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784931763373, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784931763499, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784931763500, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784932637675, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784932637676, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784932638012, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784932870493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833817839622498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932884312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382971704006195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932897562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860516250133514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932911181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874778151512146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932924637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386844664812088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932938165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871726393699646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932951528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812920451164246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932951529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932951530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784933038073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4985129237174988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784933038074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784933038074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784933051456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815785944461823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933064981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831965625286102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933078627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855266571044922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933092153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846394419670105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933105730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856039941310883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933119871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793516159057617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933134294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845030963420868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933134294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933134295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933178092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4987214505672455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933178093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933178093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933191883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823670148849487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933206044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380215436220169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933219805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803039491176605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933234151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789594173431396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933247776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809704780578613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933262169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820835947990417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933275993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781669735908508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933275994, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933275994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933318820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4990042448043823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933318821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933318821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933332715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834144175052643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933347138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797345757484436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933360948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696660101413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933375010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808898627758026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933389770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748608529567719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933403837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711605966091156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933418431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13706600666046143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933418431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933418432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933461652, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49935898184776306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933461653, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933461653, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933475097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660281896591187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933488892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740946352481842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933503091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726723194122314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933517045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366814225912094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933530966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541537523269653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933544978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679899275302887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933559105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573041558265686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933559106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933559106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933602077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.499824583530426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933602077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933602078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933615922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657966256141663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933629597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614654541015625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933643694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741682469844818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933658009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13648560643196106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933671790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464701175689697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933685432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526782393455505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933699301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718149065971375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933699302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933699302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003888010978699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742527, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933756848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13667887449264526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933770903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547274470329285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933784673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567276298999786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933798599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463151454925537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933812787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487458229064941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933827474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444650173187256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933841559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477271795272827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933841560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933841560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933884184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010992288589478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933884184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933884185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933898285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384920358657837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933912556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635583221912384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933927458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827025890350342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933941695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338326334953308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933956053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360178291797638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933970246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481006026268005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933984817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670194149017334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933984818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933984819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934028253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019844770431519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934028254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934028254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934042606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365453153848648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934056942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365528702735901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934071289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346653699874878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934085770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994441390037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934100269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324114203453064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934114328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335850954055786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934128241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482940196990967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934128291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934128292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934171466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029578804969788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934171467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934171467, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934185643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349901556968689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934199748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12905532121658325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934214768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111189007759094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934228617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686397671699524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934242697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231180608272552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934256655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122248649597168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934271248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240481913089752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934271248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934271249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934314213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503998339176178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934314214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934314214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934328081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13160935044288635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934342856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343594193458557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934356642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12794968485832214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934370487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949582934379578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934384597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337938755750656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934398999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12805138528347015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934412894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983697652816772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934412894, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934412894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934456796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052961707115173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934456796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934456797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934471711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462340831756592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934485513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309475779533386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934499660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344483345746994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934513756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350967437028885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934528278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606691360473633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934542483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353794813156128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934556974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697507977485657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934556974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934556975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934599394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5068166255950928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934599395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934599395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934613963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433733582496643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934628404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305384784936905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934642772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396002352237701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934657393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403263360261917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934672025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038834929466248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934686902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324405074119568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934701450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833078742027283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934701450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934701450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934744501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5082074999809265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934744502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934744502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934759401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257702112197876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934773886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114267587661743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934788755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729867339134216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934803416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545671105384827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934818470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245537877082825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934832690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286899447441101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934847097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12803581357002258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934847097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934847098, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934890989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.50975102186203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934890990, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934890990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934905413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018955290317535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934919836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481810688972473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934934647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14229267835617065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934949216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423112034797668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934964243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152065873146057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934978382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371416926383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934992673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360093355178833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934992674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934992674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935036793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5115908980369568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935036793, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935036794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935050926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346597969532013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935065074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642728328704834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935079551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566666841506958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935094075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315343856811523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935108753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342317759990692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935123257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281211256980896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935137491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325078308582306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935137492, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935137492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935180105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5135055184364319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935180105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935180105, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935194406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182413578033447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935208715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345825046300888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935223227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14183416962623596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935238043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338101029396057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935252760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446632027626038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935267739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227076828479767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935282328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315705955028534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935282329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935282329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935324816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5157426595687866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935324817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935324817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935338903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820959627628326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935353094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331394910812378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935367363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883338332176208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935381730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990377843379974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935396372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795489072799683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935411124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131578266620636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935425798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633495569229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935425799, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935425800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935469874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182860493659973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935469875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935469875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935484546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681994378566742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935498917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632890582084656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935513558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142946362495422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935527969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299135833978653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935542550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173484802246094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935557276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352209448814392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935572012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266731798648834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935572013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935572013, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935615473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5211758613586426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935615474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935615474, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935630033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360083520412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935645159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217535614967346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935660032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379375457763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935674665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642239570617676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935689547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384769231081009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935704249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344021111726761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935719363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13901981711387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935719364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935719364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935763234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5240886211395264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935763235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935763235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935778106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350283920764923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935793067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14776721596717834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935808395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515688478946686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935823437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312587141990662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935838455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415226340293884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935853216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311800360679626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935868217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366090476512909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935868218, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935868218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935912449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5276654958724976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935912449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935912449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935927432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604553043842316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935942613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532263040542603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935957928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337597817182541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935972375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364535570144653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935986880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332501769065857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936001277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951289117336273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936015497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12566375732421875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936015497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936015498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936058553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5321872234344482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936058554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936058554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936073040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250431418418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936087713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262292742729187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936102497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484594225883484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936117210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542290031909943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936132373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366725116968155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936147137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288546323776245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936161825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987348437309265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936161826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936161826, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936204468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5386106371879578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936204469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936204469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936219007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247206807136536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936233574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12700822949409485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936247940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331615447998047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936262372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219450414180756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936277278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705668807029724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936292175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323353350162506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936306534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426847755908966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936306535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936306535, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936350040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5471780896186829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936350042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936350042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936364982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12936198711395264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936379595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020411133766174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936394054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060204684734344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936408659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297989785671234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936422905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783095240592957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936437300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12336324900388718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936452105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712107598781586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936452106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936452106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936495927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5584678649902344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936495928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936495928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936510596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563251495361328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936525400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323497772216797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936539642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910728693008423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936554678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339997828006744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936569323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770771980285645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936583685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129603311419487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936598279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269536316394806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936598279, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936598280, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936642535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.571601152420044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936642536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936642536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936656724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912680208683014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936670951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275295913219452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936685599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248306512832642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936700397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269031047821045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936715252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13134746253490448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936729770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370569050312042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936744691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044774532318115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936744691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936744692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936789412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5860704183578491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936789412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936789412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936804375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955200672149658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936819084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997448444366455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936833854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974104285240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936848583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325576901435852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936863244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346907496452332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936877770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955273687839508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936893126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304420530796051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936893127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936893127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936938520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5988540053367615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936938520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936938520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936953342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12822359800338745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936968711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081282377243042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936983169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283353865146637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936998459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12766428291797638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937013709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272287517786026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937028796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191792368888855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937043953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231343030929565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937043954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937043954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937089440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6102829575538635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937089440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937089441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937104094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008743524551392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937119211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279986947774887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937134614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303672194480896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937150081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194531202316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937165387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336963951587677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937180220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894777953624725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937195911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288244128227234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937195912, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937195912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937241255, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6203109622001648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937241256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937241256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937256301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974829971790314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937271471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127842515707016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937286617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903723120689392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937301445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526540994644165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937316442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13076025247573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937331426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261175274848938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937346708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916278839111328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937346708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937346709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937391646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6284033060073853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937391647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937391647, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937406419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607446312904358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937421860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12815535068511963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937436743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736114859580994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937451674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713120877742767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937466377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12549060583114624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937481196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12260158360004425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937496154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12415672838687897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937496154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937496155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937540330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6366043090820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937540331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937540331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937555819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271584928035736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937570965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324894219636917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937586085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724844574928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937601390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12294614315032959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937616331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890138685703278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937631015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11959219723939896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937645692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708557605743408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937645692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937645692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937690117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.645308256149292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937690118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937690118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937705042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224801242351532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937719631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849591135978699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937734110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483287066221237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937748856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.122162364423275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937763876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030249834060669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937779067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11918897926807404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937793978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105046272277832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937793979, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937793979, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937839427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6589432954788208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937839427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937839427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937854006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725957691669464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937869560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703576147556305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937884503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12072452902793884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937899267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536988615989685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937914122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358772963285446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937928870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508932709693909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937943585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137060821056366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937943634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937943634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937987832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6728841662406921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937987832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937987832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938002377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662811785936356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938017730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442252993583679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938033010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957229673862457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938048496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457392573356628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938063278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501116305589676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938078678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12154541909694672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938093393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921785235404968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938093394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938093394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938136935, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6937421560287476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938136936, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938136936, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938151777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754634976387024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938166599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568988859653473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938181639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132710725069046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938196408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143489927053452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938211297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703767836093903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938226002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032513320446014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938241442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678753793239594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938241443, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938241443, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938285106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7171506285667419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938285107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938285107, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938300097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951511353254318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938315368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11574643105268478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938330195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11852621287107468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938345597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495689511299133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938360707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652971804141998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938375761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174336075782776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938390787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014966666698456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938390788, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938390788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938434071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7374840974807739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938434072, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938434072, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938448889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253684759140015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938463795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096355319023132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938478498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165262758731842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938493446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190599203109741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938508740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116548553109169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938524092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703637450933456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938539294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460085213184357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938539294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938539295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938582620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7497473955154419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938582621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938582621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938597659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943181812763214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938612337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231428384780884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938628256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12229709327220917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938643707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1243550032377243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938658815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11932452768087387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938674085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345656216144562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938689175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844301223754883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938689176, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938689176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938731692, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.757445752620697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938731692, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938731693, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938746765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828060448169708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938762007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048515141010284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938777510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120303675532341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938792912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310083419084549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938808588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11668585240840912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938823775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484179645776749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938839517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633064806461334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938839518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938839518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938883333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7614441514015198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938883335, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938883335, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938898638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11900435388088226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938914015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925772786140442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938929332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649004578590393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938944524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595994979143143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938959706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11939727514982224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938975038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858584940433502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938990738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523958295583725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938990739, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938990740, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939034553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7638744711875916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939034553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939034553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939050079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669066548347473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939065901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912835597991943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939080893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886730253696442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939096410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474379897117615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939111346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829667001962662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939126150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423577576875687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939140986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11572988331317902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939140987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939140988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939185042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7653255462646484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939185043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939185043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939199845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030484735965729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939214699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517201364040375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939229296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805819928646088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939243844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565771162509918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939259032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350878536701202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939274154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09713125228881836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939289573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065792441368103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939289574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939289574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939334388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661648988723755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939334389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939334389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939349323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508207976818085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939363945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016837507486343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939379932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11928801983594894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939395344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892215371131897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939410223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652267932891846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939424906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010885089635849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939440085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717801749706268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939440086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939440086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939482967, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670717239379883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939482968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939482968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939498067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498560965061188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939512930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579423606395721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939527907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966964066028595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939543151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798405855894089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939558448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649394243955612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939573758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197461187839508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939589383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973761975765228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939589383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939589384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939633872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685182690620422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939633873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939633873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939648813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513243079185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939664157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232337564229965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939679314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007751524448395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939694508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945392400026321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939709583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052856296300888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939724551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048718750476837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939739622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468053817749023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939739623, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939739623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939783010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688003778457642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939783011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939783011, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939798050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445070266723633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939813049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000535637140274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939827742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529014468193054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939842862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039353162050247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939857944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645131021738052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939873403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325074195861816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939888748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999242216348648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939888748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939888749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939932413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.76991206407547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939932414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939932414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939946984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988299012184143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939962516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336455702781677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939977673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072852611541748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939992851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364783555269241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940007771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255078971385956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940022578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735450685024261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940037591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918370634317398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940037592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940037592, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940080872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699956297874451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940080873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940080873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940095724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557430982589722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940110551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303381085395813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940125147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863751173019409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940140264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604734718799591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940155483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999766737222672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940170817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077997237443924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940185958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109069436788559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940185958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940185959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940229455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713656425476074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940229455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940229456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940244140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682470351457596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940259530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134953260421753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940274504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495256632566452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940289671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10222934186458588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940304383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814940929412842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940319210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325640976428986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940333976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837002843618393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940333977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940333977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940377334, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711402773857117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940377335, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940377335, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940392276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455602407455444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940407243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114589273929596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940422043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750517249107361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940437244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022680461406708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940452405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245441436767578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940467608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127045750617981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940483358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529027879238129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940483359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940483359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940526626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710765600204468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940526627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940526627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940541868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821934044361115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940556739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09971412271261215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940572749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643336713314056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940588404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631868779659271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940603849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490845680236816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940619397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420700162649155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940634992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106727734208107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940634993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940634994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940677882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708079814910889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940677883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940677883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940693418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149255931377411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940708792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141812652349472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940724414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002348363399506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940740109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053971946239471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940755703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146058663725853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940771129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249101907014847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940786780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261239647865295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940786781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940786781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940828983, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715572714805603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940828984, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940828984, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784940843975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953322798013687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940859676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764065384864807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed534232611.log b/recommendation_v4/rcp_logs/gbs_16384/seed534232611.log new file mode 100644 index 000000000..3069ed338 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed534232611.log @@ -0,0 +1,825 @@ +:::MLLOG {"namespace": "", "time_ms": 1784867651473, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784867651474, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784867672186, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "seed", "value": 534232611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784867672188, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784867672189, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784867672189, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784867672189, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784867672203, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784867672203, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784868550467, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784868550469, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784868550812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784868695683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831129670143127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868709143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829046487808228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868722443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861596584320068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868735746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386701464653015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868749264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138719379901886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868762716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387530267238617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868776034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855043053627014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868776035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868776035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868821115, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002787709236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868821116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868821116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868834449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816913962364197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868847924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381126344203949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868861518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856345415115356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868875078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848696649074554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868888534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382846087217331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868902346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782641291618347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868916374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845005631446838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868916375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868916375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868958959, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.50047367811203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868958959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868958960, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868972552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383132040500641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868986486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792482018470764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868999891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790208101272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869014104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791029155254364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869027543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379641443490982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869041596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790130615234375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869055073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767024874687195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869055074, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869055074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869097697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006957054138184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869097697, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869097697, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784869111321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837409019470215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869125436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781332969665527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869139119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703078031539917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869153026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802888989448547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869167549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771840929985046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869181351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137057363986969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869195636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692200183868408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869195637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869195637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869237395, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009660124778748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869237396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869237396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784869250854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664832711219788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869264599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745582103729248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869278784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728252053260803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869292493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659481704235077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869306283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352694034576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869320104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662637770175934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869334254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565444946289062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869334254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869334255, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869377252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013074278831482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869377252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869377253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784869390924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658525049686432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869404540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593320548534393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869418468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737134635448456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869432586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624969124794006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869446108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469094038009644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869459667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512760400772095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869473430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373823881149292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869473430, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869473431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869516899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016821622848511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869516899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869516899, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869531145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13667824864387512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869545193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353776752948761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869558890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552415370941162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869572891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467246294021606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869587051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479028642177582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869601598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420803844928741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869615736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473792374134064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869615736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869615737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869658136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021560192108154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869658137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869658137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869672415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396133482456207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869686819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636848330497742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869701615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821198046207428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869715852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383223116397858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869730452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360228806734085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869744720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13468918204307556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869759551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674212992191315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869759552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869759553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869802921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028121471405029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869802922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869802922, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869817377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661150634288788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869831988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662999868392944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869846437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475696742534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869860880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002508878707886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869875481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322983205318451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869889734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363084197044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869903776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497835397720337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869903826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869903827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869948043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035316348075867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869948043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869948043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869962233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490664958953857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869976320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893325090408325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869991180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312253624200821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870004994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368437111377716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870019121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216906785964966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870033022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312730461359024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870047507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248851895332336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870047507, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870047507, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870090829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.50435471534729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870090829, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870090829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784870104673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162660598754883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870119427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343628168106079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870133166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278204768896103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870146975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951980531215668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870161053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387258350849152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870175475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812435626983643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870189444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976954877376556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870189444, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870189445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870233581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053808689117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870233581, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870233581, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784870248551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467857241630554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870262512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298210501670837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870276688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448388874530792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870290816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487520813941956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870305401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627316057682037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870319624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353262841701508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870334064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687735795974731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870334064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870334065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870377357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5067327618598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870377358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870377358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784870391979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451287150382996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870406660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067777454853058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870421256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338222175836563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870436217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14044302701950073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870450994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037657737731934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870466161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259266316890717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870480622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381818950176239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870480622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870480623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870522910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.508118212223053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870522910, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870522910, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784870537584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265324354171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870552171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104583323001862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870567172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370180994272232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870581865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544104993343353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870597095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324378401041031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870611339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870019674301147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870625623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12795382738113403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870625624, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870625624, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870668941, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095651149749756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870668941, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870668941, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784870683229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13016265630722046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870697470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347208172082901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870712131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227090775966644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870726534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340966373682022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870741617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141459226608276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870756195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336064636707306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870770735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361127495765686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870770736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870770736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870814100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.511222243309021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870814100, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870814100, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784870828331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460510969161987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870842649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13628923892974854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870857149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579177856445312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870871764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317230343818665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870886540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134371817111969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870901051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329118013381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870915203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249127566814423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870915203, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870915204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870959017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5129993557929993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870959018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870959018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784870973267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198944926261902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870987478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345224678516388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871001888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14200475811958313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871016491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311918079853058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871031127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343311220407486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871046009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230592012405396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871060606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309752941131592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871060606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871060607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871105124, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5150312185287476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871105124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871105125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784871119486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384512186050415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871133883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306714594364166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871148161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880659103393555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871162575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999999523162842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871177283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789905607700348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871192155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159886002540588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871206704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636717200279236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871206705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871206705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871252537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5173589587211609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871252538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871252538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784871267193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684043288230896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871281758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633577525615692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871296683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138574361801147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871311295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009317219257355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871325924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316499412059784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871340788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352316290140152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871355768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263779878616333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871355768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871355769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871400111, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5200806856155396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871400111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871400112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784871414702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599750399589539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871429996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229453563690186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871444888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378875970840454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871459643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633385300636292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871474674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385122537612915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871489565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454076647758484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871504780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908183574676514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871504781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871504781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871548784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5228760838508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871548785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871548785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784871563796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498273491859436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871578869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15005724132061005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871594238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509711623191833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871609236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288716971874237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871624488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426116108894348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871639378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290467858314514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871654430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655880093574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871654431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871654431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871698973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5260833501815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871698974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871698974, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784871714033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598588109016418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871729065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518673181533813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871744159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388904929161072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871758678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356491923332214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871773140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312998414039612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871787615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295195370912552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871801843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255398988723755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871801844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871801844, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871845643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.529862105846405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871845643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871845644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784871860079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326170265674591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871874658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248085975646973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871889320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490593433380127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871903957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536624610424042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871918852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666260242462158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871933714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288367658853531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871948319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988066673278809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871948319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871948320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871991685, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5357736945152283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871991686, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784871991686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784872006154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271330296993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872020538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714263796806335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872034999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320419192314148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872049508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132139652967453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872064305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692728638648987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872079307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218209147453308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872093775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423976302146912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872093776, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872093776, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872137597, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5439373850822449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872137598, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872137598, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784872153028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945930659770966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872167753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037633895874023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872182130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060522079467773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872196543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979668378829956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872210776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774425745010376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872225071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12309624254703522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872239812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717992067337036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872239812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872239813, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872282216, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5549163818359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872282217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872282217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784872296798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560032844543457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872311464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324019312858582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872325734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895997405052185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872340691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356754183769226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872355233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763342261314392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872369701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975981831550598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872384396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268594115972519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872384396, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872384397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872426291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5685737729072571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872426291, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872426292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784872440659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291247010231018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872455119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275143027305603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872470037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262897729873657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872484999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273945450782776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872499972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133704662322998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872514645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354523479938507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872529865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046039640903473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872529866, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872529866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872573856, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5849584341049194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872573856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872573856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784872588761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295124888420105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872603584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12973034381866455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872618587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295442134141922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872633630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324356645345688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872648296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326729834079742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872662986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946389615535736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872678328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010287284851074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872678329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872678329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872721388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5990827679634094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872721389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872721389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784872736316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796804308891296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872751588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305992305278778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872766071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820696830749512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872781466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745344638824463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872796633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271393597126007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872811671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128598034381866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872826870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318860650062561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872826871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872826871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872869669, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6118651032447815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872869669, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872869670, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784872884341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972694635391235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872899338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760606408119202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872914623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301276981830597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872929954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196443021297455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872945316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283665478229523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872960304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12867116928100586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872975850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857729196548462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872975850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784872975850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784873019004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6231918334960938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784873019005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784873019005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784873033872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883318960666656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873048664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270534247159958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873063416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282009482383728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873078080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347387135028839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873092987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031813502311707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873108066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536409497261047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873123356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128271222114563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873123356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873123356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873167401, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6341937780380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873167402, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873167402, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784873182150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12495394051074982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873197766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273140013217926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873212706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12641362845897675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873227400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12582455575466156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873242028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12403364479541779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873256850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109233438968658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873271539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12288492918014526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873271540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873271540, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873314252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6470412611961365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873314252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873314252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784873329653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12572011351585388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873344797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12206205725669861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873359836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521092802286148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873374946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1217304915189743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873389639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11684360355138779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873404202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771039664745331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873418837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150534451007843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873418838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873418838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873461037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.66670161485672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873461038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873461038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784873475781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12070204317569733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873490187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623561382293701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873504604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198745667934418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873519297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12018879503011703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873534332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11804008483886719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873549513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11729928851127625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873564270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870730131864548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873564271, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873564271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873608046, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7062681913375854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873608046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873608046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784873622583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517713218927383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873637997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593617498874664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873652985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792518198490143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873667841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309096962213516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873682872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111665740609169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873697672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299652606248856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873712493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834610462188721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873712548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873712548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873755609, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7432421445846558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873755609, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873755609, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784873770073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349162459373474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873785368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211257427930832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873800380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11759836226701736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873815663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126935482025146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873830276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288833618164062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873845863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11800827831029892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873860600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655629634857178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873860601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873860601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873903613, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7534071803092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873903613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873903613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784873918387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514326930046082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873932991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278489977121353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873947793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013766378164291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873962355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110187828540802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873976914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412570834159851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873991294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813747137784958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874006487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366793513298035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874006488, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874006488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874048595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560184597969055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874048595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874048596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784874063524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1171932965517044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874078905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224673688411713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874093798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614972352981567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874109317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140113532543182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874124286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528624594211578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874139133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814052075147629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874153905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726726055145264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874153905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874153905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874197273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7572418451309204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874197273, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874197274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784874212082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921473801136017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874226830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078261286020279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874241325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415769159793854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874256198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890506207942963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874271260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312130093574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874286280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327710211277008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874301475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116342693567276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874301476, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874301476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874345145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7587093114852905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874345145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874345146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784874360055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793809592723846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874374827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947825014591217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874390575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11962705850601196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874406040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12237070500850677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874421078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646337807178497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874436355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11161745339632034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874451533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665044724941254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874451534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874451534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874494824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7572910189628601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874494825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874494825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784874509884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499284207820892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874525242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820601880550385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874540814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711960285902023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874556286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074027419090271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874572163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458276957273483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874587456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425580084323883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874603404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479997098445892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874603405, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874603405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874646162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7575445771217346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874646162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874646162, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784874661389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11888640373945236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874676677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727981477975845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874691949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049422174692154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874707159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451928317546844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874722226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786940693855286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874737417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068190336227417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874753020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140018105506897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874753021, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874753021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874796245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627907991409302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874796246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874796246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784874811446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537505686283112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874827065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777853429317474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874841958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897491872310638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874857359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399312317371368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874872628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072881892323494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874887516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386516153812408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874902347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11760631203651428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874902348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874902348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874943964, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7636346220970154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874943965, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874943965, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784874958808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033906042575836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874973559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608783900737762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874988262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727870464324951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875002843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496845096349716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875018005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264793038368225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875033090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09652960300445557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875048533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687272995710373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875048534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875048534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875091273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7647917866706848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875091274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875091274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784875106312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057821661233902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875121079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099134087562561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875136819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827477812767029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875151865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941098630428314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875166668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609988868236542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875181306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085061192512512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875196300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702045261859894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875196300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875196301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875239207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7655775547027588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875239207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875239207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784875254097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501977801322937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875268784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582669079303741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875283554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977998375892639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875298547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073780357837677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875313659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696415603160858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875328664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229658126831055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875344058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018851399421692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875344059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875344059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875386739, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678825855255127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875386739, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875386739, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784875401506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049475148320198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875416723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258585214614868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875431759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104254424571991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875447063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092381626367569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875462224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500769317150116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875477334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081113874912262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875492594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508377850055695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875492595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875492595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875536049, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7681392431259155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875536049, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875536049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784875551163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473819077014923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875566507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997733682394028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875581645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482582449913025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875597330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065603792667389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875612849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642364621162415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875628609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358202457427979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875644000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963509231805801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875644000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875644000, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875688013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683038115501404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875688014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875688014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784875702622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012021660804749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875718174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405413806438446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875733240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074783056974411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875748323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458119958639145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875763260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306283831596375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875778230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785192251205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875793181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974135994911194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875793182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875793182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875835666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7681777477264404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875835666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875835666, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784875850615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556681454181671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875865481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026572659611702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875880079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863825678825378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875895230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656335204839706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875910534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028216034173965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875925889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810880362987518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875941070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126358807086945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875941071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875941071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875984493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769487202167511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875984494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875984494, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784875999145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681439191102982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876014630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10141846537590027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876029450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518169403076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876044383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026071161031723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876059129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885406285524368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876073722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308582872152328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876088339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905987024307251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876088339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876088340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876132686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695403695106506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876132687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876132687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784876147517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052633747458458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876162276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102969944477081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876176907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788805782794952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876191733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042519122362137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876206575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216172575950623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876221453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279698461294174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876236886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515595972537994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876236887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876236887, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876280333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770579993724823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876280334, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876280334, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784876295162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080983430147171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876309718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041041672229767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876325641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11628971993923187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876341156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663931608200073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876356364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11525185406208038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876371632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465206205844879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876387067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690028965473175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876387072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876387073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876430377, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702311873435974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876430377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876430378, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784876445650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191637814044952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876460640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431531608104706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876475888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036457121372223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876491257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096756160259247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876506537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458972841501236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876521784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220055818557739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876537269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11326867341995239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876537270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876537270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876579995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705276608467102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876579996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876579996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784876594858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935087502002716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876610562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736905038356781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876625899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685007274150848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876641411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541114747524261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876656497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11842210590839386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876671668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989197343587875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876687220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11559885740280151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876687221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876687221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876730638, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703103423118591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876730639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876730639, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784876745665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353950202465057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876760873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387099325656891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876776145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158018484711647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876791714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422774195671082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876807303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905022919178009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876822591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919202864170074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876837528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261864006519318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876837528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876837528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876882716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712194919586182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876882717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876882717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784876898400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881486535072327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876913897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526171326637268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876928697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457506775856018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876943674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420073568820953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876958536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071244329214096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876973828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324344038963318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876988939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873030126094818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876988940, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784876988940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784877034018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719352841377258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784877034019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784877034019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784877049103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980759561061859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877063951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247887670993805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877079233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806737840175629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877094663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388046503067017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877110130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257390141487122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877125992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563518851995468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877141230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948348999023438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877141231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877141231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877186635, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719484567642212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877186636, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877186636, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784877201565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791778564453125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877217260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110660620033741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877232496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686101019382477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877247572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644641518592834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877262471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808980464935303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877277192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311993956565857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877292189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848309099674225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877292190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877292190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877338057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722658514976501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877338058, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877338058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784877352981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811396688222885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877367656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583990812301636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877382638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896550863981247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877397399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320387780666351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877412681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932959616184235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877428341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939851403236389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877444037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541057586669922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877444037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877444050, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877488268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727868556976318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877488269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877488269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784877503791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818426311016083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877518604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234349220991135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877534222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314310878515244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877549323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312984883785248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877564986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009586811065674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877580201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269467532634735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877595296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627450048923492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877595297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877595297, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877638990, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772662341594696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877638991, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877638991, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784877653988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313217341899872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877669356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066875070333481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877685013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083517074584961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877700289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992932319641113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877715351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472334921360016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877730875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928407311439514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877746320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957943648099899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877746321, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877746321, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877789945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732727527618408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877789945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877789945, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784877805600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120997816324234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877821625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529768466949463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877837142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451212525367737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877852130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595577210187912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877867604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096967220306396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877882943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076056957244873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877898470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474605858325958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877898471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877898471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877941784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729840278625488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877941785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877941785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784877956911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686597228050232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877971974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866574943065643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877987115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062025487422943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878002387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008056998252869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878017739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09696544706821442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878033206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263118147850037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878048387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382784903049469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878048387, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878048388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878093041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732160091400146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878093041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878093041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878108607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260915011167526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878124175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816407203674316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878139797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919177532196045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878155526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106003820896149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878171205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811763256788254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878186208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904116183519363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878201976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235576331615448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878201976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878201977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878245994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735798954963684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878245994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878245994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878261227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920596122741699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878277036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420659184455872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878292283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674382746219635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878307356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143476516008377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878322474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800670087337494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878337821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300487816333771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878352858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045161560177803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878352859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878352859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878396649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737305164337158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878396649, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878396650, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878411570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666278004646301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878426674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09833501279354095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878441961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088569462299347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878457295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446596890687943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878472790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152861267328262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878488387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261607706546783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878504339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403723269701004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878504340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878504340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878546872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773900032043457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878546873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878546873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878562373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297709703445435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878577500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837668180465698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed547461891.log b/recommendation_v4/rcp_logs/gbs_16384/seed547461891.log new file mode 100644 index 000000000..23b7d39e6 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed547461891.log @@ -0,0 +1,620 @@ +:::MLLOG {"namespace": "", "time_ms": 1784931743669, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784931743670, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784931764687, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "seed", "value": 547461891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784931764689, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784931764690, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784931764690, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784932551738, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784932551739, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784932552089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784932843736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14011096954345703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932858259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395670622587204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932872357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922682404518127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932886851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389314979314804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932901303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872897624969482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932915790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898824155330658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932930006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911840319633484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932930007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932930007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932977268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022149682044983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932977268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932977268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932991620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13954810798168182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933009329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393607258796692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933023724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896581530570984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933038093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13927879929542542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933052137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910746574401855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933066703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13925877213478088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933081756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890935480594635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933081757, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933081757, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933131322, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023825764656067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933131323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933131323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933145489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389566957950592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933160319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13918301463127136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933174908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894188404083252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933189849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884998857975006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933204142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138747900724411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933219703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895055651664734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933234374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872727751731873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933234375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933234375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933283392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025967955589294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933283393, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933283393, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933297849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870441913604736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933313041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384090781211853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933327991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840028643608093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933342650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386442929506302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933358506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832679390907288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933373007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827753067016602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933388725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798464834690094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933388726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933388726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933439236, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028647184371948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933439237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933439238, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933454264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137877956032753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933468794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819561898708344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933483886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790932297706604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933498814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759195804595947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933513576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691800832748413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933528447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759098947048187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933543020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695575296878815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933543021, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933543021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933593351, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032035112380981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933593352, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933593352, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933608472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739407062530518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933623112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13706108927726746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933637875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137693852186203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933652649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723309338092804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933667771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598591089248657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933682394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623571395874023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933697044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375485062599182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933697044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933697045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036138296127319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933742549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933757799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711439073085785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933773142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618260622024536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933788078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363830864429474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933803485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356608271598816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933818470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563178479671478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933833977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352245807647705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933849620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559122383594513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933849620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933849621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933899008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041259527206421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933899008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933899009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933914229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490936160087585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933929450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367165446281433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933946992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138039231300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933962917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13449686765670776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933978580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622884452342987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933993901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524246215820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934009386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666486740112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934009387, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934009387, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934060104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048261880874634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934060104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934060104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784934075592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659168779850006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934091093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674002885818481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934106676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522028923034668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934124101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065451383590698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934140039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305024802684784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934155457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338854283094406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934170929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351611614227295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934170976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934170977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934220282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055546164512634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934220283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934220283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934235448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515788316726685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934250441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980356812477112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934265975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162502646446228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934281508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665707409381866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934296728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255739212036133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934311696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316598802804947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934327123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265129923820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934327123, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934327124, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934375426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5063571929931641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934375427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934375427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934389906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318589150905609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934405797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425494730472565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934420642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12819844484329224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934436106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298903524875641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934451483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378405570983887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934466979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847624719142914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934481707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005979359149933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934481708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934481708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934528328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074077844619751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934528328, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934528329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934544320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345539391040802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934559638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301372528076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934575189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433894515037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934590580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487011194229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934606547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595494627952576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934621606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351565599441528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934637072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681206107139587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934637073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934637073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934688052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5088557600975037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934688053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934688053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934703681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436082005500793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934719353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059073686599731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934735330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370871543884277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934751138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14042264223098755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934766729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13028016686439514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934782750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234582543373108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934798854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805696368217468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934798854, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934798855, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934849002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5103106498718262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934849002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934849003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934865018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12540830671787262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934880867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105247914791107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934896856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136959046125412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934912323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542261719703674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934928325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232268393039703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934943665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871761620044708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934959253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800630927085876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934959253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934959254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935009213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5117381811141968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935009213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935009214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935024510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008713722229004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935040325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474318385124207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935055682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14225423336029053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935070883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341175138950348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935086890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152994215488434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935102322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356320559978485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935117985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595551252365112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935117986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935117986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935168130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5133688449859619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935168130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935168131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935183643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455727696418762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935199266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647542893886566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935214944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354455202817917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935230502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328252732753754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935246055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436894118785858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935261620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294774293899536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935276827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324637234210968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935276828, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935276828, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935326084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5151175260543823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935326085, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935326085, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935341649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319682002067566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935357098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443514704704285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935372668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14176282286643982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935388682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357843458652496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935403883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431233167648315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935419401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231584429740906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935435083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321059942245483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935435083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935435084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935484806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5171326994895935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935484807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935484807, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935500140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847440481185913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935515712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296541571617126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935530831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288740336894989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935546314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998417019844055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935562219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778850436210632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935577623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171282410621643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935593205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363215148448944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935593205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935593206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935644057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5194522738456726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935644058, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935644058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935659525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678738474845886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935675091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653510808944702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935690917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13130921125411987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935705911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012610375881195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935721707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316889226436615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935737308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527356088161469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935752874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265028595924377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935752874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935752875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935800966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5221157073974609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935800967, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935800967, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935816365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602040708065033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935832141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227133452892303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935848465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784542679786682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935864291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364675760269165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935880037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384248435497284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935896161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431954383850098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935912178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904334604740143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935912179, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935912179, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935961866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5247279405593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935961867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935961867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935977998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499850034713745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935993903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15196719765663147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936010368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506042957305908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936026354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306938111782074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936042597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342477798461914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936058657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133060485124588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936074841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365804672241211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936074842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936074842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936124355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5278817415237427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936124356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936124356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936140510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360626220703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936156303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352592557668686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936172430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383552432060242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936188403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336667835712433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936204158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313280045986176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936219930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295199692249298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936235560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256670504808426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936235561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936235561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936285062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5318164229393005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936285063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936285063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936300455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326139271259308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936316533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325017511844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936332187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501715660095215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936347483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542543351650238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936363155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669340312480927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936379094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288314163684845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936394783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003814220428467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936394783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936394784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936443998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5376914143562317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936443999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936443999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936459522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261525332927704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936475135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12701866030693054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936490886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322436809539795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936506827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215497136116028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936522828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270776391029358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936538853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231264054775238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936554143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433502614498138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936554144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936554144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.545604944229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936619480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955603003501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936635315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045403361320496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936651014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069327175617218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936666653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992507219314575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936682297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279032677412033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936697910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12345559149980545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936713913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719964981079102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936713914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936713914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936763766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5566655993461609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936763767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936763767, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936779871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576695322990417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936795336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321973383426666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936810239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389981359243393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936826854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369101285934448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936842537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783657014369965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936858514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297871470451355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936874264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12703537940979004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936874264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936874265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936924142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5704361796379089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936924143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936924143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936939847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12909197807312012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936955666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758800387382507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936971581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271518051624298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936987936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282476365566254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937003981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141891360282898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937019375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376379013061523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937035271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043105602264404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937035272, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937035272, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937085117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.586930513381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937085118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937085118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937101297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959027290344238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937117011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12985196709632874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937132708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298622190952301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937148703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273832201957703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937164242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362261652946472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937180120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12942543625831604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937196384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045623898506165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937196384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937196385, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937246648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6019637584686279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937246649, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937246649, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937262515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830492854118347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937278733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093438744544983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937293826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12854540348052979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937310255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275492012500763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937326945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126997172832489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937343383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156616687774658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937359958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219353556632996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937359958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937359959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937410172, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6191465854644775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937410173, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937410173, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937426436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977559864521027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937442812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279548555612564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937459462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019782304763794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937476283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163168728351593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937492489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328659534454346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937508196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289137601852417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937525157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286933720111847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937525158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937525158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937575443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.636939525604248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937575444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937575444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937591693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12914124131202698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937608158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275160312652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937624303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12886005640029907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937640416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477841019630432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937656904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037139177322388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937673381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12538450956344604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937689718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894853949546814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937689718, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937689718, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937738140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.654011607170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937738141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937738141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937755076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12531189620494843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937772297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726949155330658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937788323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265944391489029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937804795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12632550299167633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937820717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12445471435785294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937836795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12175817787647247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937852999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12302751839160919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937852999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937852999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937902775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6716793775558472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937902775, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937902776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937919341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601493299007416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937935935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12231853604316711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937951880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160428375005722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937967838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12207531929016113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937984041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11751396954059601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937999790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861905455589294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938015941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609408259391785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938015942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938015942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938066179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6885344982147217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938066180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938066180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938082228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158489227294922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938098017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11686664074659348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938114214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311318725347519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938130295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11962774395942688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938146689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878795921802521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938163473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11748354136943817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938179314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856171697378159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938179315, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938179315, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938227899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7220892310142517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938227899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938227900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938243537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535663902759552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938260457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568155884742737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938276735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11902016401290894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938292929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351892352104187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938309028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232701689004898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938324941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385143548250198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938341061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866038501262665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938341109, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938341109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938390811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7415403723716736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938390812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938390812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938406776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140742152929306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938423335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124638020992279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938439572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700350791215897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938456024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189227551221848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938471709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213544011116028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938487730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906982958316803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938503840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059151142835617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938503840, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938503840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938554273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7525176405906677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938554273, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938554274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938570255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491402238607407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938586461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135338693857193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938602744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127406358718872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938618756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037218570709229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938634980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466299951076508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938650739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861427128314972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938667072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337023228406906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938667072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938667073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938716960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597202658653259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938716961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938716961, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938733419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608533561229706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938749773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332957446575165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938765484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649259179830551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938781785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10179827362298965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938798091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411577463150024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938814191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821951180696487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938830185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684183239936829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938830186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938830186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938880076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7633603811264038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938880077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938880077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938896166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872945189476013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938912127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886016488075256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938928215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370278149843216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938944299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852843523025513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938960585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228643357753754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938976986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285025835037231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938993193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140389740467072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938993194, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938993194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939043000, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662031054496765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939043001, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939043001, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939059110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729677975177765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939074752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837195068597794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939091547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11915165185928345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939108245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12125395238399506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939124586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431124806404114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939140951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070214956998825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939157539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595958471298218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939157539, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939157540, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939207661, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678863406181335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939207662, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939207662, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939223766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453602463006973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939240092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732457786798477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939256836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698345839977264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939273156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988878458738327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939289768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324897408485413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939305857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133686825633049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939321929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377815365791321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939321930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939321930, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939371789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691175937652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939371790, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939371790, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939388538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11805646121501923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939404857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651659965515137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939421479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515613108873367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939437731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358760297298431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939453905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11824697256088257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939470648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629098117351532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939487277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352603137493134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939487278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939487278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939536074, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704080939292908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939536074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939536075, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939552689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410464346408844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939569594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681073367595673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939585336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806068778038025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939602384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327868700027466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939619053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709598660469055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939635392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391384363174438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939651455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617778241634369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939651456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939651456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939700293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711290717124939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939700294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939700294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939716616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950686037540436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939732561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449217796325684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939748458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054808497428894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939764576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441997647285461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939780760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10222119092941284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939797099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526434540748596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939813705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543382912874222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939813705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939813706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939862695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719566226005554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939862696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939862696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939878628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497322678565979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939894415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969942808151245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939911343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11802917718887329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939927784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957978665828705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939944199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590498149394989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939960077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056555271148682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939976540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616771876811981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939976541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939976542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940026257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726616263389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940026258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940026258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940042465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501104593276978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940058412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557768493890762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940074905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933056473731995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940091086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659122467041016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940107459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602819919586182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940124041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118556559085846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940140553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973838716745377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940140553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940140554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940189675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773196280002594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940189676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940189676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940205442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467787832021713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940221554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216784477233887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940238224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081535667181015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940254463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811042785644531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940270787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104738250374794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940286884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065382719039917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940303022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457825660705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940303022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940303022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940352918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735177278518677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940352919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940352919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940369160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447078943252563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940385308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955224394798279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940401505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430006682872772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940418332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000227183103561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940434905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579918324947357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940451679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031409353017807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940468157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918638110160828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940468157, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940468158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940518492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738419771194458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940518493, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940518493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784940533798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990628063678741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940550436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029524952173233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940566828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706661641597748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940583071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325615108013153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940599182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220102965831757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940615340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599412024021149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940631347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088932603597641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940631348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940631348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940680500, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741097211837769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940680501, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940680501, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784940696408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528606176376343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940712784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319405794143677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940728727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777462273836136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940745071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552187263965607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940761541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940141975879669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940778063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755304992198944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940794420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071759462356567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940794420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940794421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940844074, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744320034980774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940844075, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940844075, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784940859643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676389932632446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed574397838.log b/recommendation_v4/rcp_logs/gbs_16384/seed574397838.log new file mode 100644 index 000000000..2a2296bbd --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed574397838.log @@ -0,0 +1,1333 @@ +:::MLLOG {"namespace": "", "time_ms": 1784704984542, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784704984542, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784705004477, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784705004478, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784705004478, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "seed", "value": 574397838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784705004479, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784705004480, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784705917226, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784705917227, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784705917461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784706049170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678115606307983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706063022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729268312454224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706076655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811150193214417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706090492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855701684951782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706104384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887956738471985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706118352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869912922382355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784706132282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809779286384583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706132282, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706132283, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706218512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021247267723083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706218513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706218513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784706232258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734427094459534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706246325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374339461326599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706260481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788153231143951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706274552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771745562553406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706288853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791358470916748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706303403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13688242435455322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706318175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377115249633789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706318175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706318175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706363614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502267062664032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706363615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706363615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784706377866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770084083080292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706392455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368705928325653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706406768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723787665367126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706421406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699859380722046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706435424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733619451522827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706450125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701067864894867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706464711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369401216506958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706464712, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706464712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706509395, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024484992027283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706509396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706509396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784706523922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380326747894287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706538621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735926151275635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706553055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575127720832825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706567684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773183524608612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706582900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368788331747055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706597447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623544573783875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706612492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362687349319458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706612493, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706612494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706656696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026519894599915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706656696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706656696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784706670839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574501872062683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706685157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703618943691254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706699786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672637939453125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706714280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598471879959106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706728786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13405023515224457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706743289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603150844573975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706757778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470838963985443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706757779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706757779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706801039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029142498970032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706801040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706801040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784706815206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600413501262665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706829400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353408396244049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706844085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13707087934017181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706858704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358252763748169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706873040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364127278327942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706887561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342800110578537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706902169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717931509017944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706902169, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706902170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706945957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032214522361755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706945958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706945958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784706960732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638333976268768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706975643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13488508760929108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784706990416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507366180419922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707005206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417547941207886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707020123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415563106536865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707035289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368839025497437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707050078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413108885288239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707050079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707050080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707092199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036430358886719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707092200, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707092200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784707107111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313929736614227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707122111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361522674560547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707137477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849252462387085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707152521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327787816524506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707167613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585487008094788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707182565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344502568244934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707197799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666526973247528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707197800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707197800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707241596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042558312416077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707241596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707241596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784707256651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644909858703613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707271682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655266165733337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707286766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348104476928711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707301726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943804264068604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784707316797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320440173149109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707331626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13344067335128784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707346397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482198119163513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707346444, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707346445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707390003, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049930214881897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707390003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707390003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784707404866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494408130645752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707419602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12849479913711548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707435003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308819055557251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707449395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367919147014618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707464062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319478303194046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707478592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311306655406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707493586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230809569358826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707493586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707493587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707538573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058194994926453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707538574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707538574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784707552831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156861066818237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707568249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343226134777069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707582987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753626704216003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707597772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939167022705078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707612854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387072086334229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707628006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279432475566864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707642774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971800565719604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707642775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707642775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707688588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5069074034690857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707688589, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707688589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784707704284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473762571811676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707718850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329372525215149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707733723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435810804367065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707748617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135039821267128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707763928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361311674118042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707778745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367685675621033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707793814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700881600379944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707793815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707793815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707837528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5081657767295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707837528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707837528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784707852754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447213172912598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707868137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049235939979553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707883245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341848075389862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707898592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14056363701820374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707913877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049763441085815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707929294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239121437072754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707944508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381777822971344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784707944509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707944509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707987588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5093967318534851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707987588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784707987589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784708003047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12491695582866669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708018165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119672238826752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708033492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695210218429565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708048533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135238915681839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708064031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239309191703796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708078905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877142429351807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708093984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12801429629325867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708093985, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708093985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708136567, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107027888298035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708136568, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708136568, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784708151373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012459874153137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708166238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466057181358337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708181492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14231783151626587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708196473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388407230377197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708211739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148511946201324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708226347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371621072292328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708241090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600169122219086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708241091, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708241091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708284287, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122944116592407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708284287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708284288, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784708298850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463608920574188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708313612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630685210227966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708328695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562335073947906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708343660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332632303237915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708358771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444316387176514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708374067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294827938079834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708389032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323569416999817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708389033, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708389033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708434014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5140208601951599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708434015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708434015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784708449012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191458582878113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708464043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345173567533493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708479318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14189203083515167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708494799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404875993728638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708510102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418328762054443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708525325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321803629398346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708540237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310536742210388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708540238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708540238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160511136054993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708583294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784708598002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854734599590302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708612845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289617002010345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708627675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287558674812317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708642603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299595683813095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708657859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377996802330017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708673129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154929876327515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708688210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606473803520203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708688211, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708688211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708732283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5183751583099365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708732284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708732284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784708747542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683921098709106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708762667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643240928649902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708778070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13132993876934052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708793236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13007484376430511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708808588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149333000183105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708824005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535988330841064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708839542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259997963905334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708839542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708839542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708883126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5210455060005188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708883127, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708883127, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784708898159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359223872423172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708913731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215667009353638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708929753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784730434417725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708945346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642770051956177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708960904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387103796005249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708976400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445347547531128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708992262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13900992274284363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784708992262, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784708992263, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709035182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238339900970459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709035183, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709035183, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784709050828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496795296669006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709066397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14830750226974487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709082202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497358560562134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709097765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314715027809143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709113645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426057994365692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709129284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330876350402832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709145116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649897277355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709145116, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709145117, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709188675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5270437002182007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709188675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709188676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784709204557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361445188522339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709220388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510899245738983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709236298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367614150047302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709251466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133608877658844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709266728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331852376461029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709281843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929673492908478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709296855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12563146650791168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709296856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709296856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709341324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5310468673706055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709341324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709341325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784709356407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253550231456757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709371885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241031765937805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709387457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349334716796875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709402740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352764219045639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709418185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675656914710999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709433916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12876278162002563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709449476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12968118488788605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709449477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709449477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709492459, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5370678305625916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709492459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709492459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784709507781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255378603935242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709522939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269911229610443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709538054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311585783958435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709553163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217848539352417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709568634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697190046310425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709584229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224545121192932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709599232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341181993484497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709599232, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709599233, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709643362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5454789400100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709643362, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709643362, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784709659089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292891800403595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709674522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039928674697876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709689776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306607723236084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709705083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975266575813293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709720295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12776601314544678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709735588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12307661771774292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709751279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699627876281738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709751280, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709751281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709794974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5566381812095642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709794975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709794975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784709810599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560721278190613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709826117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328197598457336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709841144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869233429431915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709856909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339653611183167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709872358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758395075798035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709887601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295308619737625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709903058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266912817955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709903059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709903059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709947124, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5701677203178406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709947124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709947125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784709962231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288178563117981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709977493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271423101425171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784709993151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234663009643555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710008870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255274295806885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710024590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311180591583252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710039818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359948992729187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710055722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302267163991928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710055723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710055723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710099561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5860069990158081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710099562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710099562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784710115445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938192486763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710131083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952160835266113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710146800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12940382957458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710162506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238075375556946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710178020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299626111984253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710193283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907801568508148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710209281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12993918359279633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710209281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710209281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710253438, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5995160341262817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710253438, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710253439, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784710269179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783825397491455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710285072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072824478149414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710300235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774929404258728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710316268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12727266550064087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710332200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692758440971375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710347916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107740879058838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710363781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198339939117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710363782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710363782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710407164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6116767525672913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710407164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710407165, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784710422354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293150782585144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710437973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734246253967285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710453863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991347908973694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710469761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133150339126587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710485668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327056884765625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710501189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286361813545227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710517556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283879280090332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710517556, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710517557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710561112, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.622648298740387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710561112, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710561112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784710576828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287226378917694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710592591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12678758800029755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710608461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12823013961315155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710623993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485483825206757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710639764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015206158161163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710655910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12509675323963165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710671856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128437340259552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710671857, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710671857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710715873, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6338461637496948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710715873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710715874, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784710731331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12487220764160156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710747408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268545240163803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710762853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12622877955436707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710778141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12567636370658875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710793302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12427359819412231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710808579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109366059303284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710823855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12313582748174667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710823856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710823856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710867045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6502136588096619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710867045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710867046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784710882925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258542239665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710898698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12217087298631668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710914335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152331605553627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710930012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12206404656171799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710945668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169990673661232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710961166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784873902797699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710976779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531135439872742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784710976780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784710976780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784711020731, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6716891527175903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784711020732, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784711020732, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784711036583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109453231096268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711052090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673419177532196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711067526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289585381746292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711083161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12006983906030655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711099078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831097304821014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711115146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769858002662659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711130671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881951451301575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711130671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711130672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711174853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.718448281288147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711174854, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711174854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784711190009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560289561748505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711205688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11596754193305969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711221210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189248114824295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711236561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427098512649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711252186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241769790649414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711267529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350549757480621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711283043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947905480861664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711283089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711283090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711327411, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7447913885116577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711327411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711327411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784711342641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429397761821747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711358554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237627267837524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711374270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785563081502914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711390166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237527430057526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711405483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228816956281662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711421135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935360729694366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711436463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695824027061462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711436464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711436464, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711481741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7511307001113892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711481741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711481741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784711497044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11551590263843536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711512466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336953938007355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711528091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111672222614288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711543488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058039963245392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711558959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467646270990372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711574078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184108778834343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711590038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339399963617325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711590038, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711590039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711635249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7508862614631653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711635250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711635250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784711650904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681544780731201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711666790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11295398324728012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711682275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633673310279846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711698138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237440466880798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711713702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141490787267685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711729290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079946756362915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711744857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692372918128967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711744858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711744858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711789654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7482230067253113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711789655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711789655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784711805144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091657280921936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711820688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824458301067352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711836067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382021009922028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711851714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911980271339417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711867670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309777200222015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711883652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031579077243805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711899774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155291646718979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711899775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711899775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711945291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7463878393173218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711945291, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711945292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784711961134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723450779914856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711976562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090511828660965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784711992712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975656449794769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712008660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248923629522324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712024282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577610671520233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712040111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120578646659851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712055822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671298742294312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712055823, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712055823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712100626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7418042421340942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712100627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712100627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784712116218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473843455314636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712131992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080293282866478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712148091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746110022068024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712164074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082018911838531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712180246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405116319656372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712195880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381456255912781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712212051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427669435739517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712212052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712212052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712257654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.745593786239624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712257655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712257655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784712273658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11862176656723022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712289607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719950497150421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712305513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508612543344498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712321395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402912437915802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712337226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785299330949783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712353236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688035190105438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712369572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137847974896431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712369573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712369573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712414955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7529391646385193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712414955, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712414955, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784712431070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504510045051575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712447436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744623839855194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712463050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084267646074295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712479092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138271689414978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712495020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728257894515991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712510701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387499630451202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712526292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161072850227356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712526292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712526293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712571873, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7559993267059326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712571873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712571873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784712587588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980044305324554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712603294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531826853752136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712618941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645739734172821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712634423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047268956899643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712650317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220031440258026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712666153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09626621007919312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712682246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584858059883118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712682246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712682246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712727313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7604027986526489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712727314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712727314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784712743032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052447035908699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712758512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003037542104721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712774782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823403835296631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712790663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929524898529053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712806253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058986634016037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712821776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076198726892471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712837563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683576762676239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712837563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712837564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712881557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7630046606063843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712881558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712881558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784712897268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050235778093338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712912777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546886175870895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712928307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946950316429138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712943988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711351037025452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712959979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613347589969635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712975823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169468611478806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712991942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982415080070496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784712991943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784712991943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784713036318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7657108902931213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784713036319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784713036319, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784713051865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471461713314056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713067847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246274620294571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713083558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919727385044098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713099404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918228328227997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713115219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469284653663635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713130984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012458264827728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713146858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043616384267807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713146858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713146858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713191852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7657580375671387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713191853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713191853, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784713207563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434417426586151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713223274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997686743736267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713239024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477480292320251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713255280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016078293323517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713271422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576694458723068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713287784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372260212898254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713303903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099407970905304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713303903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713303904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713349958, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671083211898804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713349959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713349959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784713365343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976474940776825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713381598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291826725006104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713397410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703868418931961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713413188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340824723243713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713428849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268912464380264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713444472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694345831871033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713460252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914334654808044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713460253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713460253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713505209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670337557792664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713505210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713505210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784713520840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573878884315491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713536558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330233722925186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713552009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831698775291443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713567874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574829578399658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713583701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024101078510284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713599527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788169503211975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713615200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062557995319366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713615200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713615200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713659781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688913941383362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713659781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713659782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784713675056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647168755531311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713691410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158318281173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713707354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427691042423248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713723330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245370864868164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713739148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793409496545792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713754909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303690075874329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713770691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786759853363037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713770691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713770691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713815436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692140936851501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713815437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713815437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784713831352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396213829517365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713847316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111285001039505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713863282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747972130775452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713879396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002534836530685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713895526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184526979923248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713911502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267533898353577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713927954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496722161769867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784713927954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713927955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713972626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704147100448608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713972627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713972627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784713988600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831139981746674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714004128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967784583568573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714020693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587793380022049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714036987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561179161071777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714053134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439947038888931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714069167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378529131412506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714085422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066395491361618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714085423, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714085423, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714130052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705183029174805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714130052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714130053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784714146185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138412356376648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714162098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11412566155195236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714178296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988733917474747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714194794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026187241077423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714211229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429246515035629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714227410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209089308977127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714243893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311142146587372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714243893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714243893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714287169, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706372141838074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714287170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714287170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784714302850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950420796871185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714319183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719717293977737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714335435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700345784425735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714351926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440586298704147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714368014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830472946166992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714384242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097198873758316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714400703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598764359951019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714400704, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714400704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714443906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713290452957153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714443907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714443907, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784714459809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342689394950867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714475911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379707604646683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714492300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568190157413483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714508721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348772048950195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714525327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935530066490173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714541663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915760695934296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714557522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192916333675385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714557522, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714557523, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714601965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717194557189941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714601966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714601966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784714618347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863660275936127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714634516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449492931365967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714650114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433351993560791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714665909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042913943529129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714681666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668540000915527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714697825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300249606370926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714713719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841414332389832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714713719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714713720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714758168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772498607635498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714758169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714758169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784714773913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966675728559494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714789654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264299809932709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714805845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770389437675476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714821937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032090112566948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714838250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189671069383621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714854669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572059452533722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714870666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968750923871994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714870666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714870667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714915655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772510826587677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714915656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714915656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784714931218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757178068161011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714947619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059461534023285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714963561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061808243393898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714979330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597725957632065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784714995133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734632611274719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715010702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305216908454895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715026504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781186074018478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715026505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715026505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715070421, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730041742324829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715070422, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715070422, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784715086141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763998329639435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715101716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550148040056229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715117648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085672602057457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715133387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272519290447235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715149458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089555025100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715165574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947328805923462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715181946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489004850387573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715181947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715181948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715226976, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77330082654953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715226977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715226977, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784715243017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769224166870117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715258501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205085575580597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715274236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210760474205017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715289596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280647873878479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715305716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002216637134552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715321404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239684581756592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715337164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054937094449997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715337164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715337164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715381384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773595929145813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715381385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715381385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784715396868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314000397920609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715412788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023093223571777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715428778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833650827407837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715444527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962367057800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715459958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467614233493805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715475699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806091129779816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715491656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944290459156036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715491656, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715491657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715536352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739872336387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715536353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715536353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784715552235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130280792713165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715568429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444895923137665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715584287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398076474666595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715599766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551093518733978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715615781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999346733093262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715631996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745185613632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715648496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445526242256165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715648497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715648497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715692931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741057276725769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715692931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715692932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784715708871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658799111843109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715724749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856886208057404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715740660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016454011201859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715756526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935620963573456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715772421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09672191739082336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715788544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230182111263275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715804389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364185273647308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715804390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715804390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715847829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740169167518616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715847830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715847830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784715864024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238377749919891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715880230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827438533306122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715896397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087898388504982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715912687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109986901283264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715928926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795487463474274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715944563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903607308864594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715960735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181647539138794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784715960736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784715960737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784716005960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746319770812988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784716005960, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784716005960, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784716021708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956057906150818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716038013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041705533862114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716053954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661749541759491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716069944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177147924900055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716085986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775599628686905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716102220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231070756912231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716118341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415603220462799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716118341, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716118341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716161573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774536669254303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716161573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716161574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784716177494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662136971950531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716193589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09821304678916931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716209669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110697440803051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716225864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426183044910431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716242283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164264380931854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716258739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237410455942154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716275391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386109352111816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716275392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716275393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716319390, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748371362686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716319390, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716319390, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784716335399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296620428562164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716351089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071704626083374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716367514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10252465307712555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716383642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067979037761688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716400163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976903140544891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716416677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11869074404239655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716433088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720414459705353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716433089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716433089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716476565, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775113046169281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716476566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716476566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784716492913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813318729400635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716509313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990417003631592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716525434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484763026237488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716541793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212357133626938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716558194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146206259727478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716574852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670105159282684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716591418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853183269500732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716591419, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716591419, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716635008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750730514526367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716635009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716635009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784716651387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059731990098953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716667828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104692742228508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716683690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229506134986877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716700299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172997951507568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716716904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020301282405853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716733250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780760645866394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716749490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104561984539032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716749490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716749491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716793212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752073407173157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716793213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716793213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784716809272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11368927359580994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716825476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520327091217041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716841876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843460261821747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716858030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958436131477356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716874188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529944837093353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716890656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153604090213776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716907069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162586659193039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716907069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716907070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716950591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754827737808228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716950592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716950592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784716967005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160585880279541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716983174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500181257724762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784716999058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164824366569519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717015319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392438620328903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717031351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152419656515121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717047557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665567219257355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717063511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598799586296082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717063512, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717063512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717106960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755924463272095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717106960, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717106961, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784717122576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993970930576324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717138545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786277800798416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717154440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170702636241913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717170554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459049046039581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717186671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421943664550781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717202588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818124771118164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717218958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521373152732849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717218959, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717218959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717263094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754903435707092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717263094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717263094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784717279224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582372546195984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717295524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09982378780841827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717311837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09436718374490738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717327804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999535620212555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717343445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259014904499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717359761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530219972133636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717375882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002316325902939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717375883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717375883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717420181, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757834792137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717420182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717420182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784717436695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060321033000946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717452825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028168797492981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717469056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155717670917511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717485308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09791266918182373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717501750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703369975090027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717518075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387107729911804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717534521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077771544456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717534521, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717534521, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717579095, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759135961532593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717579096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717579096, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784717595368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10051266849040985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717611565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568572580814362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717628065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058696061372757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717644574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726279765367508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717661337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260441899299622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717677860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333190113306046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717694153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579352080821991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717694153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717694154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717740005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762230038642883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717740005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717740005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784717755964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209223628044128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717772272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591535270214081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717788131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268545150756836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717804293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117199063301086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717820127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678252577781677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717835861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027451679110527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717851850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338189244270325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717851851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717851851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717897157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765662670135498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717897158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717897158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784717912987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254387557506561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717928815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315075516700745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717944859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037703812122345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717960858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316871851682663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717977211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619707405567169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784717993625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999047756195068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718009825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09932806342840195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718009826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718009826, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718054733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766432166099548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718054733, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718054733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784718071200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129788517951965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718087486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885728359222412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718103521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069602727890015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718119084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09821663796901703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718136393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620729625225067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718152834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498849302530289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718169507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733412206172943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718169508, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718169508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718214127, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768873572349548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718214128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718214128, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784718230595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622720420360565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718246936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098812535405159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718263245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490953922271729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718279687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975077211856842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718295943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995636060833931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718312330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401482880115509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718328701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044561117887497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718328702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718328702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718373844, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770395278930664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718373844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718373845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784718390401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887871146202087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718406807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690142214298248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718423115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246307402849197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718439761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032740771770477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718456212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813398659229279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718472441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172866076231003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718488381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399800539016724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718488382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718488382, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718533687, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772161364555359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718533687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718533688, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784718550234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115639865398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718566469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955877602100372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718583080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165698826313019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718599334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507823526859283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718615483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09994281828403473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718631711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128522455692291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718647709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986527264118195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718647710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718647711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718693456, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773630023002625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718693457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718693457, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784718709695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053665280342102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718725808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201014578342438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718741718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672169923782349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718757859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401943325996399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718774008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706591606140137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718790275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319051682949066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718806531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226558148860931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718806531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718806532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718851784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777611017227173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718851784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718851784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784718868116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630898177623749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718884525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117488354444504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718900720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277942568063736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718916784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062966138124466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718933182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764588415622711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718949745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803830623626709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718966573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11574406921863556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784718966573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784718966574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784719010852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778844833374023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784719010853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784719010853, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784719027424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587279498577118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719043795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882639139890671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719060303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125369518995285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719076812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473090946674347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719093341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887718200683594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719109908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639022290706635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719126362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671916604042053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719126362, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719126363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719171801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783492207527161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719171802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719171802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784719188366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107456684112549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719205065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099162697792053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719221779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106681376695633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719238817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214964091777802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719255517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932677984237671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784719271837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144658625125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719288201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341060161590576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719288201, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719288202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719333469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783010005950928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719333469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719333469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784719350328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830318182706833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719367077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158190667629242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719383275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705292224884033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719399634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280624568462372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719415861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447066068649292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719432298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10902142524719238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719448708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339870095252991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719448708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719448708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719493996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786862254142761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719493996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719493996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784719510127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494646966457367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719526569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888584703207016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719543299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352908611297607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719560116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113106831908226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719577036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073007583618164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719593776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536563396453857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719610154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091163754463196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719610154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719610154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719655260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789868116378784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719655261, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719655261, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784719671894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031377524137497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719688014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645779967308044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719704608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326876491308212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719720926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553986579179764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719737116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040838286280632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719753223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181595176458359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719769497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834714233875275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719769497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719769498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719814087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787212133407593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719814088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719814088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784719830419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028916597366333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719846688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399628430604935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719862827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187791287899017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719878937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09789930284023285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719895202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10124550759792328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719911362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175026208162308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719927676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043524295091629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784719927677, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719927678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719972919, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791268229484558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719972919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719972920, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784719989445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814207047224045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720005844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278531908988953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720022109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09349173307418823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720038010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424105077981949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720054558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224610567092896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720070763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235967487096786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720087326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545244812965393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720087326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720087327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720132360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792197465896606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720132361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720132361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784720148737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769806265830994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720165040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561825335025787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720181083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869885981082916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720197356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922777116298676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720213706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201633721590042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720229694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258819907903671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720245788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421102494001389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720245789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720245789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720290092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794784903526306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720290092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720290093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784720306031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274870693683624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720322403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906291007995605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720338650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09648561477661133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720354670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386062413454056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720371142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309275984764099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720387800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828941524028778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720404050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647307336330414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720404051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720404051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720447714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798039317131042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720447715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720447715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784720463955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023845300078392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720480051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449281334877014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720496720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582105815410614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720513053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980867803096771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720529653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592561215162277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720546154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781943798065186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720562372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304506123065948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720562373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720562373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720607409, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798337936401367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720607409, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720607409, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784720623720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407781600952148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720640018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161325335502625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720656308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015048623085022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720672546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991407260298729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720688920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509677976369858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720705086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267267912626266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720721485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019439846277237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720721485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720721486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720766184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800476551055908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720766184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720766184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784720782298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630983114242554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720798484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053580641746521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720814937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027563214302063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720831378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887555778026581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720847458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434967279434204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720863063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846536815166473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720879264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09820818901062012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720879265, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720879265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720923452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803586721420288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720923452, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720923452, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784720939302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387272387742996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720955755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972662940621376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720972012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777513891458511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784720988267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820606350898743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721004701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245595872402191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721020875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056051105260849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721037393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348232626914978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721037394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721037394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721082548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806719541549683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721082549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721082549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784721098698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114560067653656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721114992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455549508333206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721131173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168935358524323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721147225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995468497276306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721163652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539674758911133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721179971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849665105342865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721196428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09198581427335739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721196429, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721196429, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721241160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808188199996948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721241160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721241161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784721257862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232578217983246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721274203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204920172691345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721290692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876034200191498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721306778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049613356590271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721323009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587653517723083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721339271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045297309756279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721355693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935158491134644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721355693, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721355694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721400378, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812017798423767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721400378, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721400379, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784721416769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11559431254863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721432954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642871260643005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721449179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182337462902069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721465444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053429543972015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721481799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736563801765442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721498205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840614140033722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721514542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961773455142975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721514543, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721514543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721558974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814204096794128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721558974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721558974, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784721575309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279487609863281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721591797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038765013217926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721608075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230474919080734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721624838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426436364650726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721641251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633537173271179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721657672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237658768892288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721674278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021578311920166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721674279, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721674279, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721717982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814531922340393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721717983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721717983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784721734557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529261618852615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721750924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850804299116135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721767398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748592019081116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721783411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760371387004852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721800257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381923615932465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721816967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471384227275848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721833675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847542971372604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721833676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721833676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721877165, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820051908493042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721877165, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721877166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784721893604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528337001800537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721910068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080276370048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721926457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410458385944366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721943103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464365780353546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721959631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12182991951704025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721976195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422177404165268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721992527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403868347406387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784721992527, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784721992528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784722035962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820935845375061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784722035963, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784722035963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784722052470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11641409993171692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722069234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435239017009735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722085899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425276100635529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722102817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115696281194687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722119691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982206463813782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722136631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152428686618805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722153061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032983660697937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722153062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722153062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722197184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822748422622681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722197185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722197185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784722213391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646227538585663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722230465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479295790195465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722247317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073316633701324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722263772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156004667282104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722280586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504267156124115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722297028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433908343315125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722313786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755968630313873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722313787, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722313787, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722358261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825287580490112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722358262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722358262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784722374811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632236838340759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722391584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11647965013980865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722408240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138010025024414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722425499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300849914550781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722442465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482895165681839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722459500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598774790763855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722476585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004891991615295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722476586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722476586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722520353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783000111579895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722520353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722520353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784722537113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150289922952652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722553644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547352373600006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722570629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785512626171112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722587031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593065619468689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722603790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698173940181732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722620205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911960154771805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722636748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137877404689789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722636749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722636749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722681499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7830920219421387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722681499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722681500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784722697963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644946992397308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722714597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488349944353104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722730964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371200531721115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722747588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261331290006638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722764020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462117940187454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722780683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716857016086578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722797335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090645045042038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722797336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722797336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722842214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832556366920471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722842215, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722842215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784722858749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340990662574768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722875489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492715775966644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722892137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123977810144424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722908344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139471083879471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722924665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388767719268799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722941620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979323089122772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722958016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549436509609222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784722958061, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784722958062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784723003062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834515571594238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784723003063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784723003063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784723019382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098406672477722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723035605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161283612251282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723051687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006360650062561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723068060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875153750181198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723084854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101140096783638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784723101061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259610414505005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed782855750.log b/recommendation_v4/rcp_logs/gbs_16384/seed782855750.log new file mode 100644 index 000000000..8e2af9256 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed782855750.log @@ -0,0 +1,676 @@ +:::MLLOG {"namespace": "", "time_ms": 1784926157114, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784926157115, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784926177270, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "seed", "value": 782855750, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784926177272, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784926177527, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784926177527, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784927081968, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784927081969, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784927082288, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784927239499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376626044511795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927253440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137804314494133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927266659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822005689144135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927280306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868525624275208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927293812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388828605413437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927307185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384182721376419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784927320462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826918601989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927320463, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927320464, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927364889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012149810791016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927364890, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927364890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784927378231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137703537940979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927391658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790380954742432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927405206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381080448627472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927419029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797149062156677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927432618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384325921535492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927446785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375143676996231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927461028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385776549577713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927461029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927461030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927504050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014086961746216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927504053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927504053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784927517717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808190822601318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927531897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746735453605652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927545401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375974863767624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927559717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722708821296692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927573295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769273459911346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927587528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376458704471588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927601134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737203180789948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927601134, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927601134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927642832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016529560089111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927642833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927642833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784927656517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825973868370056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927670901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374104768037796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927684752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363711953163147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927698893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379597783088684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927713456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370789110660553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927727281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367053985595703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927741792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13680648803710938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927741792, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927741793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927784415, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019574165344238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927784415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927784416, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784927798091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610446453094482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927811966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710689544677734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927826151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369991898536682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927840039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620547950267792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927853878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469505310058594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927867886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642048835754395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927882172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499832153320312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927882173, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927882173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927925441, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023584365844727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927925441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927925442, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784927939175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613854348659515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927952873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567084074020386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927966950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723990321159363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927981332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360432654619217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784927995206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340222805738449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928009145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459084928035736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928023263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370900273323059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928023263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928023264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928066540, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028505921363831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928066541, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928066542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784928080951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639488816261292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928095070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497158885002136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928108770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538265228271484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928122673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421303033828735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928136773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440155982971191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928151340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413332402706146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928165790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433876633644104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928165791, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928165791, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928207801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503443717956543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928207802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928207802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784928222409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341385126113892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928237040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623614609241486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928251915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831913471221924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928266044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340383768081665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928280343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588359951972961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928294380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458402454853058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928308962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674728572368622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928308963, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928308963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928350687, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041977763175964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928350688, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928350688, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784928365254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652122020721436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928379969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669651746749878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928394644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440662622451782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928409319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955646216869354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784928423767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209909200668335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928437761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334480196237564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928451739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346956193447113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928451794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928451794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928494526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5050453543663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928494526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928494526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784928508732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349819004535675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928522769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12853291630744934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928537830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308046281337738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928551814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700565695762634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928566031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203813135623932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928580125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109256327152252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928594837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233496248722076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928594837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928594837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928637113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5059986710548401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928637114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928637114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784928650917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13155719637870789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928665712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417267799377441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928679320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276014745235443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928693137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12936876714229584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928707138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133804053068161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928721387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12798909842967987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928735114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960490584373474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928735114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928735115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928778510, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072160959243774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928778510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928778510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784928793508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467201590538025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928807511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330549418926239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928821748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445687294006348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928835842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501986861228943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928850511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617941737174988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928864756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352349400520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928879215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708555698394775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928879215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928879216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928921815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086470246315002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928921815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928921816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784928936179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435092568397522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928950650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053834438323975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928964812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390398025512695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928979322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14052040874958038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928993788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035690784454346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929008544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132512629032135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929023185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815763592720032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929023185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929023186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929066086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5099921226501465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929066087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929066087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784929080917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13022896647453308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929095515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312026083469391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929110451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701950013637543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929125044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535574078559875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929140099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323889195919037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929154284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880530953407288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929168611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796637415885925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929168611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929168611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929211498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114412903785706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929211498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929211499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784929225782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300477385520935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929240062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475026190280914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929254726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14221519231796265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929269080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133914053440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929284220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159146904945374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929298372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337442398071289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929312844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600462675094604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929312845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929312845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929355530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5130937099456787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929355531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929355531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784929369740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465572893619537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929384168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362747848033905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929398862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356944441795349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929413534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321858644485474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929428207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436144590377808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929442589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133013516664505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929456809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259616494178772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929456809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929456810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929501299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.514907956123352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929501299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929501300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784929515425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319291591644287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929529478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445520401000977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929543814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14180389046669006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929558337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430514931678772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929572909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341383159160614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929587645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237512111663818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929602206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331511288881302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929602207, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929602207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929644887, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5170245170593262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929644889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929644889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784929659094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832110166549683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929673495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306251168251038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929687738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889474630355835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929702193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002699613571167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929716974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379118263721466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929731808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315631866455078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929746621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363416612148285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929746622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929746622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929790405, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5194810628890991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929790406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929790407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784929804767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674049079418182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929818929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645029067993164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929833241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159671425819397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929847402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002312183380127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929861803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315188705921173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929876416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534659147262573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929891214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259369134902954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929891215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929891215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929933638, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5224257707595825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929933638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929933638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784929948124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593228161334991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929963152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132416233420372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929977944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797831535339355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929992672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644713163375854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930007563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849592208862305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930022241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448002934455872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930037309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390131413936615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930037309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930037309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930079860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.525466799736023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930079861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930079861, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784930094774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500447571277618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930109721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1478569507598877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930125050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350925713777542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930140053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308417797088623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930155093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424824178218842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930169738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314279913902283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930184570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650301098823547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930184570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930184570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930227292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5292485952377319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930227293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930227293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784930242133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361599564552307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930257036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353200376033783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930272229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337677240371704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930286733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336369514465332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930301201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332125961780548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930315616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918256223201752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930329924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536895275115967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930329924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930329925, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930372380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5336562395095825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930372381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930372381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784930386856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324237883090973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930401526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246741890907288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930416186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493463397026062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930430859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531288504600525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930445814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675585389137268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930460494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288081705570221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930474999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297157108783722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930475000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930475000, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930517472, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.54000324010849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930517473, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930517473, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784930531759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229872286319733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930546021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269683688879013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930560272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330803632736206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930574557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132076233625412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930589247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699133157730103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930604174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218840956687927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930618493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396461308002472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930618494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930618494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930662169, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5488908290863037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930662169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930662169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784930677281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938162684440613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930691818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302758753299713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930706118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044190406799316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930720467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964442372322083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930734577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274692863225937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930748901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1230480819940567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930763732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12688817083835602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930763733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930763733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930806729, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5613940358161926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930806730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930806730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784930821362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532491028308868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930836067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300873339176178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930850284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386261284351349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930865392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296610116958618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930880047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272779107093811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930894525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903501093387604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930909110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12632277607917786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930909111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930909111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930951899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.576807975769043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930951900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930951900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784930966078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286076009273529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930980267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686917185783386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930994881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188570737838745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931009712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193008303642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931024584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069120049476624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931039056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290004432201385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931054191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975597381591797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931054192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931054192, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931097083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5942441821098328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931097083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931097083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784931111906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289774477481842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931126710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12882760167121887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931141729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852494418621063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931156557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13161206245422363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931171285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322091519832611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931185703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281801164150238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931200993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903060019016266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931200994, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931200994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931244978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6104823350906372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931244979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931244979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784931259862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12673602998256683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931275047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916792929172516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931289556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12656454741954803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931305227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568411231040955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931320296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252669245004654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931335260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958884239196777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931350428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303296834230423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931350429, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931350429, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931393800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6419984698295593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931393800, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931393800, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784931408382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780433893203735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931423387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12548179924488068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931438574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127826988697052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931453865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956543266773224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931469189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308910846710205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931484086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263713538646698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931500098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12667182087898254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931500099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931500099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931542732, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6867356300354004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931542732, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931542732, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784931557793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12676723301410675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931572965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12467384338378906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931588126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12563641369342804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931603212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204491138458252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931618383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272323727607727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931633715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12234009802341461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931649244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249765157699585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931649244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931649245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931693019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7143635153770447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931693020, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931693020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784931707949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12200212478637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931723579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12376454472541809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931738539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12325460463762283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931753395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.121815986931324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931768132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12027117609977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931783039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709046363830566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931797904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11992722749710083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931797904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931797905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931841193, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7354652881622314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931841194, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931841194, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784931856461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12160679697990417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931871565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890718340873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931886634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067476868629456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931902059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711336672306061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931917061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246077716350555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931931749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391161382198334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931946528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132499575614929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931946529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784931946529, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784931989880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.746183454990387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784931989882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784931989882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784932004924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11642338335514069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932019490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121250689029694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932033912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799799859523773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932048558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384722590446472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932063504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130281537771225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932078558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227796971797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932093260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344914346933365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932093260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932093260, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932136267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7545377612113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932136267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932136269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784932150809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096874475479126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932166147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112702488899231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932181194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162407249212265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932196050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995270311832428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932210948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958635807037354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932225692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939804464578629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932240418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362482815980911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932240464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932240464, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932284505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7571631669998169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932284505, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932284506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784932299133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810085386037827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932314377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085176020860672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932329334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147146075963974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932344655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824751853942871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932359338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979347676038742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932374674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573080718517303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932389524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305166244506836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932389525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932389525, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932433381, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.762530505657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932433382, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932433382, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784932448248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135169118642807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932463105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082189530134201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932478167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825502127408981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932493000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748501867055893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932507898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183258354663849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932522548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465819180011749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932538046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073217540979385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932538047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932538047, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932581305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649196982383728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932581306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932581306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784932596460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143011748790741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932611775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169569194316864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932626472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422830820083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932641953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009552538394928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932656975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371153593063354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932672050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618472099304199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932687244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506321489810944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932687245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932687245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932731591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7665733098983765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932731592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932731592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784932746575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807368159294128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932761621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769647359848022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932776409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227276921272278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932791389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810402035713196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932806759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206204444169998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932822086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135897994041443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932837542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944380611181259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932837542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932837543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932881266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685302495956421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932881267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932881267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784932896531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689255595207214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932911276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799267888069153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932926671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822985112667084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932942062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12149415165185928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932957247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408329010009766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932972553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092279851436615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932987785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496765911579132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784932987785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784932987786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784933031416, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699291110038757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784933031417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784933031417, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784933046445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392492055892944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933061697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789236426353455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933077187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11697103083133698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933092518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059319972991943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933108127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342525482177734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933123267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397464573383331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933138892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043422520160675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933138893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933138894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933183258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705758213996887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933183259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933183259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784933198608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812915652990341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933213982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709266364574432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933229309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444661974906921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933244471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367730796337128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933259650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11804324388504028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933274933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702274739742279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933290633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424438655376434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933290633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933290634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933334633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708227038383484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933334634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933334634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784933349853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514782905578613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933365434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792386531829834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933380513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849332064390182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933396111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113242968916893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933411083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705579817295074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933425781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460804402828217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933440507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721976101398468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933440507, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933440508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933484081, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697142362594604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933484082, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933484082, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784933498864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947300493717194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933513759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463305354118347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933528656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655726492404938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933543404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523119568824768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933558624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196909308433533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933573783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09681645035743713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933589149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642533749341965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933589149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933589150, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933633948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709494233131409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933633949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933633949, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784933648994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506248474121094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933663708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052156984806061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933679280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765065044164658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933694560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927781462669373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933709487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630400478839874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933724175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094514489173889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933739140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662305355072021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933739140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933739141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933782524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711796164512634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933782524, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933782524, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784933797484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564565658569336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933812194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519096255302429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933827100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930485278367996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933842061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732044279575348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933857228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633505880832672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933872392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251448094844818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933887729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939338058233261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933887729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933887730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933932363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718886137008667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933932364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933932364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784933947343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510868579149246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933963031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272912681102753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933978103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919620096683502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933993229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929697751998901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934008186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10481821000576019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934022972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133959352970123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934038011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486127436161041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934038012, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934038012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934081730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722771167755127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934081730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934081730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784934096602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471920669078827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934111429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061214119195938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934126092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508856922388077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934141370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025913059711456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934156497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636846721172333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934171861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346599668264389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934187084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022360622882843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934187084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934187084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934230846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725766897201538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934230848, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934230848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784934245428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990881383419037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934260953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338803380727768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934276029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723128914833069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934291084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359860956668854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934306091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336162149906158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934321152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632757842540741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934336269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970734059810638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934336269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934336270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934379929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729324698448181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934379929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934379930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784934394935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621669888496399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934409935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465019196271896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934424743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823123157024384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934440164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622413456439972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934455604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965773463249207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934471266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765855014324188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934486631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103485524654388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934486632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934486632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934530039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727112770080566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934530040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934530040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784934544931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715138167142868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934560574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237397998571396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934575550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045914888381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934590684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197058320045471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934605397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873720049858093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934620426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320950090885162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934635409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830367356538773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934635409, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934635410, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934678414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731378674507141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934678415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934678415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784934693490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378158092498779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934708513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151833832263947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934723284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794644057750702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934738645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065540671348572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934754063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221560835838318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934769208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292291432619095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934784951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513786971569061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934784952, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934784952, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934828054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735145092010498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934828055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934828055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784934843147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085008978843689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934857866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012203454971313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934873834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163325309753418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934889494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565996706485748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934904839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486536264419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934920172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462883114814758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934935764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659930109977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934935764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934935765, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934981039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735799551010132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934981040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934981040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784934996418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129443347454071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935011661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391884088516235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935027075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983163118362427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935042839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994210839271545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935058494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142735704779625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935073947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201508343219757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935089720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304529011249542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935089721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935089721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935133133, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737796306610107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935133134, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935133134, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784935148311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945701599121094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935164244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779841244220734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935179688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776568949222565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935195319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430446803569794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935210673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861900985240936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935225904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985022783279419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935241611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11575767397880554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935241612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784935241612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed786517729.log b/recommendation_v4/rcp_logs/gbs_16384/seed786517729.log new file mode 100644 index 000000000..be264b697 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed786517729.log @@ -0,0 +1,585 @@ +:::MLLOG {"namespace": "", "time_ms": 1784931506857, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784931506857, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784931528138, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "seed", "value": 786517729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784931528140, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784931528183, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784931528183, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784932164888, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784932164889, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784932165099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784932666784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898473978042603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932682231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884752988815308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932697018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877776265144348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932712363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878664374351501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932727432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872966170310974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932742697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879486918449402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932758056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889682292938232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932758058, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932758058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932843367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015324354171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932843368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932843368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932858308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886049389839172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932873638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880683481693268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932889134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867619633674622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932904062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860756158828735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932919025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386345624923706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932934991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856877386569977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932950946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793890178203583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932950947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932950947, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933002248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017310976982117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933002249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933002249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784933017430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858480751514435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933033491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835549354553223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933048416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838523626327515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933064892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840821385383606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933080002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819941878318787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933096566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854914903640747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933112071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803303241729736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933112071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933112072, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933161989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019874572753906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933161989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933161990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933177084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384793072938919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933193793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792727887630463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933209322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374504268169403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933224853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382223665714264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933241925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378718614578247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933257055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761654496192932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933273314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376309096813202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933273314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933273315, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933324002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023024678230286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933324002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933324002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933339724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725417852401733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933354997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787530362606049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933370835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762032985687256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933386650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715656101703644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933402164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13616037368774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933417441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721799850463867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933433799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365022212266922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933433800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933433800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933483976, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026879906654358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933483977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933483977, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933499764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693612813949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933515465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654008507728577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933531065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375579833984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933547824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684746623039246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933563592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354392021894455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933579201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13578388094902039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933594516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756683468818665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933594517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933594517, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933644831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031211972236633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933644832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933644832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933661515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693571090698242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933677496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588222861289978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933693093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609427213668823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933709010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519829511642456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933724527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540229201316833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933741041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485506176948547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933756921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352388560771942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933756921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933756921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933805795, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036556124687195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933805796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933805796, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933821697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345565766096115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933837211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649317622184753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933854032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802382349967957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933870361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444282114505768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933886725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615509867668152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933902452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350935697555542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933919120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367815136909485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933919121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933919122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933969671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5043526291847229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933969672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933969672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933985956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653649389743805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934002305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136667400598526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934018283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510417938232422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934034238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308104693889618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784934050871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327565461397171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934067000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396023213863373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934083065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501469790935516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934083112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934083113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934134221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5051441788673401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934134222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934134222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784934150292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495121896266937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934165833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129506915807724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934182847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151252269744873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934198528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366274505853653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934214250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251417875289917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934230003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147050142288208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934245949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260000944137573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934245950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934245950, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934296418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060450434684753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934296419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934296419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934311595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205093145370483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934328543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419216871261597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934344433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12842640280723572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934360441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298254132270813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934376658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337822526693344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934392714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12839040160179138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934408109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991291284561157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934408109, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934408110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934459882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072081089019775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934459883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934459883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934477356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471117615699768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934493019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314950466156006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934508992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434743881225586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934524893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348819136619568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934541146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360463798046112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934556553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336071789264679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934573226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699758052825928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934573227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934573227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934624167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086670517921448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934624168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934624168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934640598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442812860012054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934657052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051560521125793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934673666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338217407464981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934690364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14031001925468445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934706460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045236468315125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934723827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243144750595093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934740216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806472718715668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934740216, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934740217, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934790455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100842118263245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934790456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934790456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934807095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697471678256989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934823474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114649057388306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934839600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695114850997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934855513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537457585334778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934872602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323358118534088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934888347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288674771785736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934904255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811116874217987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934904256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934904256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934953704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5116221904754639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934953705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934953705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934969661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300649791955948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934985716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465812802314758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935001647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14210990071296692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935017117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403496146202087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935034462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163360953330994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935050366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133737251162529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935066319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360279619693756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935066319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935066320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935115914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132721662521362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935115914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935115915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935131728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466346263885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935147763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13616493344306946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935164249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559803366661072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935180146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331765353679657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935196141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443078100681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935212632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290005922317505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935228796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262605667114258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935228797, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935228797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935280162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5150278806686401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935280162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935280163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935296356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319141983985901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935312530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442307710647583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935328787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14154626429080963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935345241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339891105890274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935361055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424262404441833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935377803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243839144706726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935394253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329737901687622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935394253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935394254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935443293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5170847177505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935443293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935443293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935459155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824062049388885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935475103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303279876708984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935491143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881925702095032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935507376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300082802772522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935523778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378546804189682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935539857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316823810338974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935556597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637246191501617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935556598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935556598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935607369, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5193681120872498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935607370, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935607371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935623658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678336143493652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935639698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362813413143158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935656206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131459578871727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935672310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003166019916534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935688634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165007531642914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935705155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498163223266602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935721585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264836370944977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935721585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935721585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935772871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5219409465789795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935772872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935772872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935788763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588649034500122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935806232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227200508117676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935823132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798919320106506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935839715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642480969429016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935856304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851426541805267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935872695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422739505767822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935889456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909171521663666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935889456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935889457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935939746, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.524542510509491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935939746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935939747, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935956337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349862813949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935972625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15065997838974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935990458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512060046195984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936007000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311581313610077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936023704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429217040538788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936040294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299143314361572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936057106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654303550720215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936057106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936057107, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936107646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5275810360908508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936107647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936107647, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784936124383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362769901752472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936140649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521543145179749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936158236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336124837398529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936174446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358289003372192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936190758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326311111450195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936207164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129564106464386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936223597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255851686000824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936223597, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936223598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936273227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5310937166213989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936273228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936273228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784936289497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13267964124679565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936306102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252723217010498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936322615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348501443862915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936338649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352558135986328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936355557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366642713546753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936372179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889301776885986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936388449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298687607049942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936388450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936388450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936438669, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.535902202129364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936438670, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936438670, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784936454851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258537650108337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936471108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12707959115505219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936487375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133267343044281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936503668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224291801452637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936520009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712039053440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936536250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224922120571136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936551718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428017497062683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936551719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936551719, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5423170924186707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936603182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936620980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12940585613250732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936637445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305665373802185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936653836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307133287191391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936670174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299382448196411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936686383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279589831829071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936702659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234017089009285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936719384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273295283317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936719385, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936719386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936769132, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5506282448768616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936769133, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936769133, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936785729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357661485671997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936802196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333463668823242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936817808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911879062652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936835912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362883031368256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936852365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12785381078720093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936868856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297951191663742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936885326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712877988815308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936885326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936885327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936935387, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.561124324798584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936935388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936935388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936951669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911637127399445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936968114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771821022033691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936984801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328277587890625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937001495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329396665096283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937018089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315653920173645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937034029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392771780490875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937051710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307498812675476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937051711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937051711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937102289, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5725361704826355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937102290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937102290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784937119148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972219288349152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937135819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981605529785156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937152617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017243146896362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937169310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329658031463623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937185785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338839828968048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937202199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987929582595825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937219220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307077705860138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937219221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937219221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937269020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5840134024620056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937269021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937269021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784937285586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12858983874320984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937302255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148248195648193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937317868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921035289764404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937335677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813618779182434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937352560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277923583984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937369403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218730688095093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937386389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307993113994598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937386390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937386390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937436456, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5943934321403503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937436456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937436457, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784937453202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064652681350708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937470258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12866172194480896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937487460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109326362609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937504526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324712336063385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937521299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340145766735077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937537722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974077463150024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937555782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12953686714172363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937555783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937555783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937605791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6046862006187439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937605791, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937605792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784937622513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306086927652359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937639361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12839025259017944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937656297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300029754638672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937673188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580678403377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937690241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147014379501343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937707377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693928182125092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937724167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050317764282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937724168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937724168, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937775052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6115285158157349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937775053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937775053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937791340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721037864685059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937809434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286616027355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937826165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283898800611496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937842879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12862449884414673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937859511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12658122181892395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937876218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366801500320435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937892889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12507563829421997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937892889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937892890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937943196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6178302764892578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937943198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937943198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937960442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281648874282837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937977502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408845871686935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937994026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821454018354416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938011180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240721195936203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938028049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12009420245885849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938044498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207176148891449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938060938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11820967495441437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938060939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938060939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938111141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6246562004089355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938111141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938111142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784938127829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240130215883255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938144108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11959535628557205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938160527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634513735771179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938176963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12332018464803696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938193555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12095258384943008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938210428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12020768225193024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938226622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162234842777252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938226622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938226623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938277064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6342114210128784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938277065, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938277066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784938292841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782489717006683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938310579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750458925962448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938327262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12126891314983368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938343818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163719892501831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938360378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517143249511719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938376814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594276875257492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938393307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241300404071808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938393358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938393359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938443928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.642349123954773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938443929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938443929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784938460332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757209151983261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938477284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515772342681885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938494058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11960060894489288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938511040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569858342409134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938526826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156340017914772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938544052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299385666847229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938560672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031246185302734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938560672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938560672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938611034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6545653939247131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938611035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938611035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784938627704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841578781604767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938644248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599033325910568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938661018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386166512966156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938677581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233580112457275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938694212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786126554012299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938710509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12134946882724762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938727509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755618453025818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938727510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938727510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938777258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6704023480415344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938777259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938777259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938793892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977432668209076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938810730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733097583055496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938826770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963599920272827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938844025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657193511724472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938860812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167156919836998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938877538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119871437549591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938894368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080241203308105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938894369, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938894369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938944503, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6930586099624634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938944503, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938944503, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938961153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334621161222458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938977997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251559108495712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938994649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11710500717163086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939011664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273345351219177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939028811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11845217645168304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939045946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806006193161011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939063026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153797134757042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939063027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939063027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939114085, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7182535529136658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939114086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939114086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784939130856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107901930809021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939147015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131569892168045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939165149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12368859350681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939182328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536616623401642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939199391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049229443073273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939216452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390930414199829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939233480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919798702001572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939233480, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939233480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939283983, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7428810596466064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939283984, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939283984, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784939300815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11933280527591705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939317874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197298765182495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939335276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12183603644371033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939352355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406227946281433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939369641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11837802827358246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939386307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11625337600708008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939403493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819266736507416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939403494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939403494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939455497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7528859376907349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939455498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939455498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784939472727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966514587402344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939489901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117392033338547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939507019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078840121626854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939523779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623630672693253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939540647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207611933350563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939557793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039076745510101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939575154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161867007613182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939575154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939575155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939624764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592704892158508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939624765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939624765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784939641756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11740115284919739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939658824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094895750284195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939674861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063157021999359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939692652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500294506549835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939709271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927718132734299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939725805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579325258731842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939742270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626521497964859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939742270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939742271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939791487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7620676755905151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939791488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939791488, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784939808138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041397601366043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939824764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547105759382248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939841378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907407104969025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939858003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659777373075485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939875070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496875643730164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939892104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689071774482727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939909246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677805542945862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939909247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939909247, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939959679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656545639038086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939959680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939959680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939976523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584641247987747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939992811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035020649433136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940011037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195593848824501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940028189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028608679771423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940045122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698390752077103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940061764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10153816640377045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940078640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690774768590927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940078640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940078641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940128224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7674455642700195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940128224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940128224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784940145120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568457841873169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940161673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576844215393066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940178525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993097722530365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940195362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764259099960327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940212467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069626659154892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940229513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176281422376633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940246794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006252467632294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940246795, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940246795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940296908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689560055732727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940296909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940296909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784940313105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515223443508148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940329967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270000994205475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940347033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904045403003693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940364002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916228592395782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940380809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536027699708939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940397512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078658163547516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940414310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508742928504944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940414310, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940414311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940464837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7700018286705017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940464837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940464838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784940481319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551631450653076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940497942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986080229282379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed803889169.log b/recommendation_v4/rcp_logs/gbs_16384/seed803889169.log new file mode 100644 index 000000000..9476b64e4 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed803889169.log @@ -0,0 +1,1353 @@ +:::MLLOG {"namespace": "", "time_ms": 1784718508646, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784718508646, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784718527972, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "seed", "value": 803889169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784718527973, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784718528026, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784718528027, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784719166965, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784719166966, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784719167171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784719304337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390523612499237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719318058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388501524925232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719331909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873723149299622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719345904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870353996753693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719360031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864946365356445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719374392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876891136169434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784719388585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387384533882141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719388586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719388586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719433130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022972226142883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719433130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719433131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784719447101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388857662677765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719461349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386702060699463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719475640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868966698646545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719489942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867343962192535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719504341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386623978614807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719518975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385720670223236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719533674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384349763393402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719533674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719533675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719577892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024787187576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719577892, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719577892, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784719592407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860026001930237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719607001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834749162197113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719621315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834577798843384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719636176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824811577796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719650552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384546458721161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719665380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382480263710022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719679829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808271288871765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719679829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719679830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719724460, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027028918266296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719724460, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719724461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784719738878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851448893547058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719754043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810987770557404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719768876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760298490524292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719783799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818247616291046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719799047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785387575626373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719813597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377972960472107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719828656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13743987679481506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719828657, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719828657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719872911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029817819595337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719872912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719872912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784719887668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722531497478485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719902487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378052532672882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719917304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762414455413818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719931919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13724291324615479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719946552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13607898354530334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719961122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712239265441895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719975722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13619504868984222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784719975723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784719975723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784720020482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033083558082581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784720020483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784720020483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784720035214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694316148757935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720049940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366569995880127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720064888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375298947095871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720079605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367347091436386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720094133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540756702423096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720108621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575372099876404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720123173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759252429008484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720123174, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720123174, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720168272, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036553144454956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720168273, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720168273, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784720183062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370520144701004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720198123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575688004493713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720213050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360742449760437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720228067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351906955242157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720243086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537010550498962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720258252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486450910568237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720273300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520705699920654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720273301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720273301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720316920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041154026985168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720316920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720316921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784720331849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437822461128235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720346767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662150502204895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720362016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821586966514587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720377177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410572707653046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720392461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622058928012848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720407564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498471677303314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720422642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659608364105225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720422643, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720422644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720467335, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047582983970642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720467336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720467336, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784720482433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660430908203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720497643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663652539253235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720512822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507090508937836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720527900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055455684661865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784720542892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281726837158203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720557743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384531438350677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720572352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503021001815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720572405, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720572405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720616596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054590106010437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720616597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720616597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784720631455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350078582763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720646131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295129954814911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720661178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138344883918762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720675707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671058416366577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720690500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239499926567078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720705252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157698512077332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720720374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253431022167206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720720374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720720375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720764323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062825679779053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720764324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720764324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784720778708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181641697883606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720793875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420993089675903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720808474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824687361717224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720823125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12957406044006348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720837852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375240564346313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720852917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128381609916687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720867433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992101907730103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720867433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720867434, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720911477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074487328529358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720911477, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720911478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784720926538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345025897026062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720941114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307929039001465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720955989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344081163406372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720970816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489562273025513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784720986081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361265778541565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721000817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346286118030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721015590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682737946510315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721015590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721015591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721060086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090480446815491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721060087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721060087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784721075221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432525098323822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721090696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036221265792847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721106177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393200933933258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721121715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14038397371768951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721137021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042956590652466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721152330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240262866020203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721167524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810104131698608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721167524, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721167524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721211594, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5106326341629028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721211595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721211595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784721227151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769201397895813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721242549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310761570930481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721258000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703018426895142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721273202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534967601299286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721288588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323331594467163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721303655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871646881103516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721319095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12793110311031342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721319096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721319096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721363035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122409462928772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721363036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721363036, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784721378288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301274299621582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721393665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347433477640152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721409178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1421850621700287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721424348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134195476770401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721439593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139688968658447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721454416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369929790496826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721469513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359732449054718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721469513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721469514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721513901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5140767693519592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721513901, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721513902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784721528745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134589284658432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721543836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363328993320465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721559338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355687379837036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721574755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133085697889328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721589838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419881463050842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721605297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296031951904297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721620553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256165385246277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721620553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721620554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721663848, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160124897956848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721663848, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721663848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784721678898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200627267360687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721693974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343829333782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721709276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14185689389705658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721724871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476096093654633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721740374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340646743774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721755623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242128491401672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721770917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310250639915466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721770918, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721770918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721814188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5183067917823792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721814189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721814189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784721829560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795071840286255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721844935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330115795135498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721860261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288823038339615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721875688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005316257476807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721891334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784128427505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721907002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151703774929047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721922184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362605094909668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721922185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721922185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721966982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5208142399787903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721966983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721966983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784721982482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686370849609375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784721997912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643664121627808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722013450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128966093063354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722028768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000941276550293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722044257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139887154102325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722059957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523942232131958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722075781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326705515384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722075782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722075782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722119907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238367915153503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722119908, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722119908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784722135256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590773940086365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722150966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222824037075043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722166736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789910078048706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722182456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362209916114807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722198351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384010761976242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722214073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435612618923187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722230161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909050822257996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722230162, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722230162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722275357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5269607901573181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722275358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722275358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784722291264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505956530570984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722307161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14679580926895142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722323092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508281111717224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722339012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293758034706116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722355074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341703236103058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722370825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284799456596375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722386748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641753792762756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722386749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722386749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722431201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5305635929107666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722431201, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722431201, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784722447195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360931098461151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722462984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533049821853638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722478723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370807468891144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722494117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350558280944824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722509473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316738605499268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722524789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948790192604065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722540020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553173303604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722540021, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722540021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722584927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5347399115562439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722584928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722584928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784722600371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242758810520172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722615970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239943981170654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722631544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490024209022522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722647037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532647490501404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722662348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656187057495117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722678083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287839114665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722693576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297197937965393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722693577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722693577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722738161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5410400629043579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722738161, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722738161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784722753588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249143958091736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722769040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271311342716217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722784554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302499055862427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722800169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210885226726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722816044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269637644290924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722831968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212203979492188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722847331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415095210075378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722847332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722847332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722892803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5499864816665649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722892804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722892804, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784722908439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293686032295227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722923961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019992411136627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722939428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050153851509094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722954915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972044944763184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722970150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276628077030182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784722985516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12315213680267334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723001274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12700039148330688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723001275, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723001275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723045381, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5615397691726685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723045382, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723045382, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784723060797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551929593086243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723076350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307034969329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723091478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387125551700592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723107284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328012824058533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723122948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12759727239608765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723138444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294931173324585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723154105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659883499145508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723154106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723154106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723198658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5750980973243713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723198659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723198659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784723214037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883900105953217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723229532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733806669712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723245362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219070434570312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723261389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250693678855896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723277360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311759203672409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723292880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339129090309143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723308402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13014408946037292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723308402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723308403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723352065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5899373292922974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723352066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723352066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784723367709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12923134863376617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723383201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12950006127357483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723398660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902358174324036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723414141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322569102048874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723429513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327083259820938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723444740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291511356830597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723460696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008032739162445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723460696, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723460697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723505328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6016857028007507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723505329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723505329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784723520935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275852918624878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723536782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130485400557518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723551899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753818929195404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723567965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273450255393982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723584347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686514854431152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723600412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087710738182068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723616598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167059421539307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723616599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723616599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723659605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6119763851165771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723659605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723659606, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784723675306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934130430221558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723691423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711289525032043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723707881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977400422096252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723724159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168638944625854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723740301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325710117816925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723756052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12841099500656128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723772059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12836244702339172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723772060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723772060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723815623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6216235160827637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723815624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723815624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784723831375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289774775505066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723847265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705320119857788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723863126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12773168087005615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723878702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443449139595032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723894476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301208734512329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723910432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1250869333744049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723926375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796461582183838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784723926375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723926376, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723971064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6298657655715942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723971064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723971065, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784723986546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12487674504518509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724002784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758871912956238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724018504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606820464134216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724034220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12569333612918854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724049649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12442421913146973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724065090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114951014518738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724080508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12325599044561386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724080509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724080510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724123772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6392695307731628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724123773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724123773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784724139702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259945183992386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724155565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12241348624229431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724171230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552789062261581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724186647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12225696444511414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724202319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785619705915451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724217713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848830431699753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724233080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576742678880692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724233080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724233081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724276544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.651831328868866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724276545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724276545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784724292165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12142974138259888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724307523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176995262503624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724322743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322980374097824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724338235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226675808429718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724354049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199483796954155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724369811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11902622878551483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724385260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081910133361816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724385260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724385261, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724429266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6663236618041992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724429266, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724429266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784724444482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709339171648026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724460513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700358986854553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724476225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12005665898323059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724491659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11522425711154938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724507085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353608965873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724522321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464214324951172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724537634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170372366905212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724537684, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724537684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724580390, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6843106150627136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724580390, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724580390, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784724595500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614332348108292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724611378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431874334812164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724627197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12045003473758698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724643249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463892459869385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724658579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601287871599197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724674294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12053036689758301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724689527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102297455072403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724689527, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724689528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724733331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7019452452659607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724733332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724733332, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784724748687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11793700605630875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724764058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531615257263184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724779699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129547655582428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724795093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266811192035675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724810586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804346203804016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724825769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12087342143058777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724841781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781731992959976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724841781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724841782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724885217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.71845543384552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724885218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724885218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784724900903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12122289836406708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724916887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153930202126503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724932336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11900125443935394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724947939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063040941953659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724963649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11768296360969543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724979330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282731592655182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724994973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057417094707489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784724994974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784724994974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725037999, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7311462759971619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725037999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725037999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784725053409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316868662834167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725068960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203798651695251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725084220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708870530128479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725099810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325110495090485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725115599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11697155237197876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725131300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784853249788284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725147213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657627671957016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725147214, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725147214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725189462, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7400932312011719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725189462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725189462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784725205135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140729486942291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725220548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376457661390305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725236610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12451612204313278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725252657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12673529982566833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725268492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12120756506919861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725284527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539237201213837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725300433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977110803127289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725300433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725300434, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725343517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7441847920417786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725343518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725343518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784725359230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965851485729218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725375252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120695024728775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725391474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12210661917924881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725407632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458833515644073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725423908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975344270467758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725439575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11710778623819351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725455407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880248248577118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725455408, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725455408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725499002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7476539015769958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725499003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725499003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784725514831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12086735665798187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725530823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133447289466858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725546693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670046508312225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725562506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746825277805328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725578382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12124906480312347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725594375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110112071037292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725610841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11738571524620056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725610841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725610842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725653921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7559205293655396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725653921, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725653921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784725669988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769688874483109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725686292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162959039211273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725701919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129388213157654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725717596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609230935573578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725733290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968497395515442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725748788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583549737930298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725764156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11650621145963669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725764157, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725764157, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725807340, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7577565312385559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725807341, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725807341, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784725822957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166330426931381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725838559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11629930138587952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725854060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227557808160782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725869411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070525199174881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725885285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048966646194458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725901141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09892280399799347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725917247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827575623989105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725917248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725917248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725960674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606504559516907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725960674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725960675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784725976332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732267796993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784725991637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065878719091415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726007994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12159538269042969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726024009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977181792259216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726039704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732895135879517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726055049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282759368419647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726070697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872595012187958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726070697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726070698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726114448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7610953450202942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726114448, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726114449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784726130025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547864437103271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726145411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770297050476074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726161086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041879653930664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726176663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850059986114502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726192521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074829176068306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726208314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263226717710495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726224305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148113012313843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726224305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726224305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726268639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7617540955543518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726268639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726268640, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784726284170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651957988739014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726299786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274983942508698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726315560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037150025367737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726331401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945439338684082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726346969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532931238412857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726362482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007247269153595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726378205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472682118415833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726378205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726378206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726421303, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7613145709037781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726421304, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726421304, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784726436783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425564646720886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726452509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023986548185349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726468020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531425476074219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726483976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106280982494354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726499791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106552854180336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726515792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399836301803589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726531645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033733189105988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726531645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726531646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726574954, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627428770065308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726574954, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726574954, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784726590130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998762920498848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726605845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379237681627274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726621518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749106854200363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726637098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048876941204071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726652622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021476536989212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726667993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815134644508362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726683575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903064906597137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726683575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726683576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726726355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628162503242493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726726356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726726356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784726741834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614549368619919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726757358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029953807592392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726772744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947231948375702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726788540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675263404846191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726812613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030014604330063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726834473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803112387657166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726857460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122530698776245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726857465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726857466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726901151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651209235191345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726901151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726901151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784726916378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623662918806076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726935182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103116929531097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726951036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486719012260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726968928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187637060880661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784726984580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834425687789917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727001301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300167441368103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727016982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861711204051971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727016983, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727021749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727063756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649539113044739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727063757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727063757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784727079557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485458374023438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727095315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053668707609177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727110976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794192552566528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727126869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100522980093956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727142819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272501945495605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727158742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130276620388031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727174981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530925542116165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727174981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727174982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727217634, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7672056555747986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727217635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727217636, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784727233470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838152468204498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727248938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996268168091774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727265116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560660600662231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727281542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155121847987175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727297577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480718851089478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727313528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463976860046387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727329722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689745098352432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727329722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727329722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727371880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677083611488342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727371881, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727371881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784727387983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111941710114479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727404042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392922699451447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727420298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018303036689758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727436787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094126105308533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727453154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404999345541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727469398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217768490314484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727485888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278559267520905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727485889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727485889, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727530227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679795026779175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727530228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727530228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784727546027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10879379510879517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727562074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069520115852356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727578277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687307268381119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727594805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442161351442337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727610818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775188893079758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727626866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012467741966248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727643287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526647955179214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727643288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727643289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727686665, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685266733169556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727686666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727686666, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784727702713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420920491218567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727718851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282798647880554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727735187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558730900287628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727751541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398781836032867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727768002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895553231239319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727784266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878886282444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727800107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089980602264404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727800107, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727800108, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727844168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769229531288147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727844168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727844169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784727860038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087651327252388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727876171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491664707660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727891747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486388206481934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727907447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457822680473328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727923141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692471265792847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727939252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029764711856842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727955058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839636623859406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784727955059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784727955059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784727998312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713470458984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784727998313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784727998313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784728014104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952776670455933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728029732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218502581119537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728045784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745237022638321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728061797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284270346164703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728077921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221205651760101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728094230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571612417697906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728110059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943531453609467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728110060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728110060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728154302, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771411657333374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728154302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728154302, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784728169783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074303388595581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728185981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105521097779274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728202162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622557252645493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728218224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613907873630524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728234264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729673504829407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728250071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10332653671503067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728266060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760782659053802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728266060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728266061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728309305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726250290870667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728309306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728309306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784728325291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791368782520294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728341202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515071451663971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728357328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832835733890533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728373261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297199338674545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728389490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084987074136734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728405810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917229950428009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728422279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451546311378479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728422280, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728422280, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728466344, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728912234306335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728466344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728466345, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784728482650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743865370750427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728498311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188774764537811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728514233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178989708423615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728529804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281847417354584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728546081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998542532324791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728561906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10193581879138947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728577700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566350817680359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728577700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728577701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728622607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731733918190002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728622608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728622608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784728638206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030646562576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728654294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000518873333931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728670511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847225785255432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728686568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928893089294434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728702225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042746976017952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728718369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081262081861496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728734454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910243541002274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728734455, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728734455, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728777452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738591432571411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728777453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728777453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784728793717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11054211109876633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728810320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420765727758408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728826527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393781214952469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728842226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548366606235504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728858266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036160051822662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728874183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720208287239075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728890644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405413806438446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728890644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728890645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728935217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742528915405273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728935218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728935218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784728951202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695576667785645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728967088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768810659646988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728983023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034225881099701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784728999021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892686247825623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729015052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661240875720978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729031257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202871263027191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729046851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320933163166046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729046851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729046852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729089928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737529873847961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729089929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729089929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784729106184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234060138463974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729122359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822752118110657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729138423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084246039390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729154766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100640520453453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729171103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766290128231049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729186872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934347659349442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729202845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149212181568146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729202845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729202846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729245959, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750225067138672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729245959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729245959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784729262126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921581089496613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729278526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414236783981323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729294563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655064880847931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729310519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123634874820709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729326424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743696987628937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729342579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240220069885254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729358573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371364653110504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729358574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729358574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729403248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749373912811279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729403248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729403249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784729419171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611330717802048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729435212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09792156517505646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729451257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036402732133865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729467353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410649329423904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729483689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114671379327774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729500008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271827667951584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729516545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361406952142715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729516546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729516546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729561185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752889394760132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729561186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729561186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784729577200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233916342258453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729592920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686949640512466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729609395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10195275396108627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729625774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009397357702255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729642483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103205680847168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729659034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11846470832824707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729675503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11766298115253448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729675503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729675504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729719895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757650017738342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729719896, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729719896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784729736332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11789388209581375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729752860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958980023860931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729769057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453508585691452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729785412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140819638967514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729801957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448876559734344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729818676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11637459695339203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729835407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811048746109009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729835407, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729835408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729879595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750682830810547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729879596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729879596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784729896399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065700650215149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729912987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460060834884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729929044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137206852436066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729945449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191797256469727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729961881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945475846529007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729978157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719314962625504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729994544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965609550476074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784729994544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784729994544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730038242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774948000907898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730038243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730038243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784730054449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280408501625061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730070834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480460524559021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730087545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788847506046295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730104154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910025238990784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730120671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489630490541458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730137448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111407727003098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730153975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540211737155914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730153976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730153976, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730197066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758489847183228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730197066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730197066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784730213823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598198860883713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730230244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404958367347717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730246382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164322942495346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730262795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385856032371521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730279236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10153228044509888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730295622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634193569421768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730311970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585927963256836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730311970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730311971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730354722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758554220199585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730354722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730354722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784730370858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026285588741302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730387074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741571336984634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730403567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134261846542358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730419919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488282144069672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730436168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423272848129272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730452164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09793966263532639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730468597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543755441904068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730468598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730468598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730512119, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762970924377441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730512120, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730512120, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784730528426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057351753115654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730545142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967593848705292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730561871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09407258033752441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730578369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998416543006897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730594456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171022802591324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730610600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043933853507042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730626578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005619168281555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730626579, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730626579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730669733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766962647438049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730669734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730669734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784730686341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053791269659996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730702529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240234434604645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730718515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486268043518066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730734502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0975942611694336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730750736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718424618244171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730766842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380083322525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730782981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034935176372528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730782981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730782982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730825684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766362428665161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730825685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730825685, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784730841661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002180427312851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730857776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537664592266083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730874225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599758476018906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730890693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076703816652298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730907317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234256088733673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730923698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311084240674973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730939951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521750152111053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.865833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784730939951, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730939952, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730982859, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767080068588257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730982859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730982860, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784730998607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157783329486847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731014939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564866662025452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731031252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270634293556213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731047578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123847961425781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731063772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678718984127045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731079617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304736346006393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.965833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731095826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288927495479584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731095826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731095827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731138971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770130634307861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731138971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731138972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784731155049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224159061908722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731170977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287649929523468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731186884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999222993850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731202633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263115167617798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731218715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579152405261993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.065833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731235028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949715226888657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731251241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0987078845500946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731251241, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731251242, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731295843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777108907699585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731295844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731295844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784731312164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096532106399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731328508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0984276831150055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731344553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014566779136658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731360136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784939140081406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.165833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731376463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590891540050507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731392512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419508814811707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731408654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692042112350464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731408654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731408655, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731453415, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773790955543518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731453415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731453416, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784731469378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563509166240692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731485269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837815165519714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731500989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459919273853302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.265833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731516977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919635951519012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731532798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996703952550888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731548701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403575003147125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731564680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372591018676758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731564680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731564681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731608141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77744060754776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731608142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731608142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784731624140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777683019638062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731640254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738794505596161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731656373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249939560890198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731672784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110219806432724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731689041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760065913200378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731705013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120840460062027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731720753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304437577724457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731720753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731720754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731765237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775139212608337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731765239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731765239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784731781303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109181702136993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731797353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929428040981293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731813834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10025452077388763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731829936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474307835102081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731845856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931440651416779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731861799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098860576748848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731877591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09947013854980469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731877592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731877593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731921292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776626944541931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731921293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731921293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784731937509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485856980085373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731954655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178034007549286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731970428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628293454647064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784731986439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371878743171692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732002481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677438974380493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732018809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130642294883728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732035005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021890789270401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732035005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732035006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732077831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777687311172485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732077832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732077832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784732096836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066158264875412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732113164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105064511299133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732129316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027524322271347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732145342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10624788701534271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732161349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726308822631836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732177683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772638022899628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732194260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433997005224228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732194261, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732194261, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732238364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782884240150452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732238365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732238365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784732254848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532569885253906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.815833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732271359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796800255775452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732287994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11194942891597748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732304498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423560976982117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732321202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888529568910599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732338009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580956935882568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732354583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653418302536011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732354583, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732354584, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732399639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781241536140442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732399640, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732399640, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784732416445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110199242830276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732433389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029176414012909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732450187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988758504390717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732467298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196576803922653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732484283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891179740428925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784732500802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396321654319763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1.0015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732517205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322140157222748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732517205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732517206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732561950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782864570617676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732561951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732561951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784732578904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776640474796295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1.0049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732595898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115226149559021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732612438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061759814620018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1.00825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732628971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278054863214493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0099166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732645492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384168267250061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1.0115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732662268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862746834754944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.01325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732678958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347614973783493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1.0149166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732678958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732678959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732722998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784984111785889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732722998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732722998, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784732739344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11532709002494812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732755953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860724002122879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1.01825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732772728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362588405609131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0199166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732789448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091751605272293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1.0215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732806243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697639733552933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.02325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732822791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478414595127106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1.0249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732838804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046688258647919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732838805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732838805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732883901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781712412834167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732883902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732883902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784732900346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001100093126297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1.02825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732916637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626315325498581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0299166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732933382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371638834476471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1.0315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732949856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555684566497803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.03325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732966246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388609021902084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1.0349166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732982413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072899401187897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732998760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834247827529907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1.0382499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784732998760, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784732998761, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733042628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784311175346375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733042628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733042628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784733058878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986448287963867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733075182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424336791038513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1.0415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733091274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10088085383176804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.04325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733107398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09753648936748505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1.0449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733123735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089342296123505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733139964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138905048370361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1.0482499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733156152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389858484268188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733156152, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733156153, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733201128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782261967658997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733201128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733201128, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784733217780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857610404491425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1.0515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733234088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271921753883362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.05325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733250320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09216845780611038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1.0549166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733266081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402827709913254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733282193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206253826618195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1.0582499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733298167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019136905670166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733314431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491164028644562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1.0615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733314432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733314432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733360105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77864009141922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733360106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733360106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784733376141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09776505827903748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.06325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733392229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052480936050415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1.0649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733408081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083431988954544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733424251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885672479867935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1.06825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733440514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184574127197266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733456546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207248479127884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1.0715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733472543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451863706111908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.07325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733472544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733472544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733517236, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778857946395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733517237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733517237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784733533071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248737037181854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1.0749166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733549591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962264239788055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.076583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733565914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09647436439990997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1.0782499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733581914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036107987165451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733598390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297323763370514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1.0815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733615014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09844411909580231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.08325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733631296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600442439317703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1.0849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733631297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733631297, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733674875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791138291358948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733674876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733674876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784733691040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167054831981659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0865833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733706895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367664694786072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1.08825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733723178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568075627088547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733739514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975217282772064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1.0915833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733755890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463470965623856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.09325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733772146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771060734987259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1.0949166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733787981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343889892101288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.096583333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733787981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733787982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733831791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793381214141846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733831792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733831792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784733847830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369203984737396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1.0982499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733864025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120244324207306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0999166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733880231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121618211269379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1.1015833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733896356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09908410161733627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.10325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733912352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478067398071289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1.1049166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733928259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257591843605042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733944476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214415192604065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1.10825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784733944477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784733944477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784733988354, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794325351715088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784733988354, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784733988355, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784734004529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583549737930298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1099166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734020718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491068661212921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1.1115833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734037192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102378711104393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.11325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734053767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782058537006378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1.1149166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734069963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413624346256256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734085922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09849926829338074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1.1182499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734102092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09803474694490433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1199166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734102092, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734102093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734145748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793163657188416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734145748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734145749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784734161788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406550765037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1.1215833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734178612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0973050594329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.12325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734195087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09759830683469772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1.1249166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734211376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814547538757324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734227926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227794945240021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1.12825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734244262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554040968418121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1299166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734260993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133560836315155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1.1315833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734260993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734260994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734304253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795529961585999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734304253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734304254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784734320760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095888912677765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1332500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734337133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435716062784195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1.1349166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734353459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097211599349976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365833333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734369548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100517064332962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1.1382499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734386130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504601150751114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1399166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734402476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799048840999603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1.1415833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734418903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09116865694522858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.14325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734418904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734418904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734463123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793929576873779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734463124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734463124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784734479869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214358568191528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1.1449166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734496318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173705220222473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734512755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866668820381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1.14825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734528891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040407419204712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1499166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734544950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686729848384857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1.1515833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734561395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413295030593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1532500000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734578143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904969483613968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1.1549166666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734578144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734578144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734621834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798340916633606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734621835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734621835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784734638457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622126400470734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734654947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661369562149048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1.1582499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734671369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230319201946259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1599166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734687783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106029748916626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1.1615833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734704225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707777738571167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1632499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734720949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888388752937317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1.1649166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734737551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992068275809288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734737551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734737552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734782177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799350023269653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734782178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734782178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784734799014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319194734096527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229703680, "lr": 1.16825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734815857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032496839761734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1699166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734832395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262706130743027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230359040, "lr": 1.1715833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734849399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430803149938583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.17325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734865961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618540644645691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231014400, "lr": 1.1749166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734882572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243329405784607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734899502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099577397108078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231669760, "lr": 1.17825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734899503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734899503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734945318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798921465873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734945319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734945319, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784734961967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538331210613251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1799166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734978624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082068681716919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232325120, "lr": 1.1815833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784734995408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755395144224167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1832499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735011586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734444856643677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232980480, "lr": 1.1849166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735028368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325433313846588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735045335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473500937223434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233635840, "lr": 1.18825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735062395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11826424300670624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1899166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735062395, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735062396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735107695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802773714065552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735107695, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735107696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784735124571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472838371992111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234291200, "lr": 1.1915833333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735141462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838501155376434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.19325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735158189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438743770122528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234946560, "lr": 1.1949166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735175237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046493873000145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735192085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12161421775817871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235601920, "lr": 1.19825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735209077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403752863407135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1999166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735225866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403299123048782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236257280, "lr": 1.2015833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735225867, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735225867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735270286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804437875747681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735270287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735270287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784735287140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11619822680950165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2032499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735304125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396980285644531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236912640, "lr": 1.2049166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735320952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359626054763794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2065833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735337846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143413931131363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237568000, "lr": 1.20825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735355017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059422791004181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2099166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735372216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538569629192352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238223360, "lr": 1.2115833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735388903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005118489265442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.21325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735388903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735388903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735434989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805465459823608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735434990, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735434990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784735451473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589397490024567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238878720, "lr": 1.2149166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735468539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114352285861969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735485566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104709729552269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239534080, "lr": 1.21825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735502343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118701696395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2199166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735519279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492198705673218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240189440, "lr": 1.2215833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735535835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399495601654053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2232499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735552627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11759863793849945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240844800, "lr": 1.2249166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735552627, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735552628, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735597569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806946039199829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735597570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735597571, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784735614402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643226444721222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2265833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735631345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645737290382385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241500160, "lr": 1.22825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735647931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130307614803314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2299166666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735665146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298063397407532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242155520, "lr": 1.2315833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735681996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581367999315262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2332499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735699007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162201315164566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242810880, "lr": 1.2349166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735716143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952402651309967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735716143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735716144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735760677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805962562561035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735760678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735760678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784735777558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422137171030045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243466240, "lr": 1.23825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735794553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554843932390213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2399166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735811633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724194347858429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244121600, "lr": 1.2415833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735828349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533390939235687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.24325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735845248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724429786205292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244776960, "lr": 1.2449166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735861859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914549231529236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2465833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735878455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081696301698685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245432320, "lr": 1.24825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735878456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735878456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735921733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804448008537292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735921734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735921734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784735938155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604891180992126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2499166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735954882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528513044118881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246087680, "lr": 1.2515833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735971417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391547322273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2532499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784735988335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231239140033722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246743040, "lr": 1.2549166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736005016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443706065416336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736021726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727232694625854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247398400, "lr": 1.25825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736038533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979050397872925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2599166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736038533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736038534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736083626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806668877601624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736083627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736083627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784736100398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330112814903259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248053760, "lr": 1.2615833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736117330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154743880033493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.26325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736134158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015772894024849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248709120, "lr": 1.2649166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736150502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994978249073029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2665833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736166875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104033462703228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249364480, "lr": 1.26825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736183846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986549407243729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2699166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736200634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561409592628479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250019840, "lr": 1.2715833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736200732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736200733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736244336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803975939750671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736244337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736244337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784736261167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055485367774963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2732499999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736277899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101424939930439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250675200, "lr": 1.2749166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736294186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134632885456085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736310651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912469238042831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251330560, "lr": 1.27825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736327463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10073123872280121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2799166666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736343975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266347229480743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251985920, "lr": 1.2815833333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736360617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923762619495392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.28325e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736360722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736360723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736405532, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808695435523987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736405533, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736405533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784736421865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08319468796253204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252641280, "lr": 1.2849166666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736438295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439413785934448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2865833333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736455346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947485268115997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253296640, "lr": 1.28825e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736472027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633162945508957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2899166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736488637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11579447239637375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253952000, "lr": 1.2915833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736505464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051582396030426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2932499999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736522442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363534837961197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254607360, "lr": 1.2949166666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736522559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736522560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736566599, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809459567070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736566600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736566600, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784736582993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10193581134080887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784736599306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305796563625336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255262720, "lr": 1.2982499999999998e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed851663607.log b/recommendation_v4/rcp_logs/gbs_16384/seed851663607.log new file mode 100644 index 000000000..a1d1eb9c5 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed851663607.log @@ -0,0 +1,676 @@ +:::MLLOG {"namespace": "", "time_ms": 1784917043413, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784917043413, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784917062951, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "seed", "value": 851663607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784917062952, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784917062953, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784917062953, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784917730633, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784917730634, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784917730919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784917881157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394873559474945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917894810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13934031128883362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917908407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389835774898529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917922214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865821063518524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917935986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873766362667084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917949920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882599771022797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917963719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867926597595215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917963720, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917963720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784918008028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000302791595459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784918008029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784918008029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784918021860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13917016983032227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918035738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909979164600372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918049833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894298672676086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918063954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899695873260498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918078165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874197006225586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918092549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888561725616455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918107083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871243596076965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918107083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918107084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918150405, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001891851425171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918150406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918150407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784918164637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387714445590973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918179016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865219056606293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918193230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863739371299744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918207779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865487277507782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918221920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860197365283966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918236435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802596926689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918250777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846050202846527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918250777, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918250777, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918294492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003810524940491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918294492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918294493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784918308854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858389854431152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918323667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831177353858948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918338177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805046677589417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918352846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845577836036682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918367785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797816634178162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918382260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783526420593262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918397084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769042491912842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918397085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918397085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918440773, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006298422813416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918440773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918440774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784918455234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741980493068695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918469760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776931166648865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918484214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775129616260529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918498694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729546964168549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918513175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363067924976349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918527774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721315562725067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918542317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646860420703888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918542318, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918542318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918586114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009397268295288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918586115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918586115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784918600518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710501790046692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918614893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675719499588013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918629519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780446350574493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918644098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369369477033615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918658487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135428786277771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918672809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588789105415344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918687269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375715732574463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918687270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918687270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918731711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013230443000793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918731712, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918731712, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918746287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693250715732574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918761118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360560655593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918775570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625293970108032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918790262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525302708148956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918805128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532063364982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918819978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497918844223022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918834626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511772453784943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918834626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918834626, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918877790, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018174648284912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918877790, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918877791, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918892803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344814896583557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918907742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644108176231384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918922851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807934522628784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918937841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417133688926697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918953096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613376021385193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918968056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350381076335907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918983199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671445846557617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918983200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918983200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784919027520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024912357330322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784919027521, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784919027521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784919042670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645566999912262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784919057795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652709126472473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784919072817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494575023651123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784919087755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060414791107178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784919102792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327623873949051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919117769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339285671710968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919132591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500714302062988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919132640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919132640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919176267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032829642295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919176268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919176268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784919191342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512451946735382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919206189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952083349227905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919221191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141974806785583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919235823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664230704307556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919250673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241037726402283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919265453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13140922784805298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919280721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132548987865448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919280722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919280722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919325136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041992664337158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919325136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919325137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784919339668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317163109779358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919354754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341826617717743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919369258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813124060630798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919383874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970700860023499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919398781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375481963157654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919413840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282307505607605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919428342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986043095588684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919428343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919428343, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919473706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054285526275635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919473707, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919473707, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784919488969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346060037612915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919503754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331506371498108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919518830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344548761844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919533852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494114577770233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919549288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623285293579102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919564223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358330726623535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919579153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689161837100983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919579153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919579154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919623157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5069516897201538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919623158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919623158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784919638187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433921337127686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919653361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13057872653007507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919668453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395926356315613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919683733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1404644101858139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919698781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025525212287903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919714026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256536424160004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919729365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815239071846008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919729365, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919729366, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919773556, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5082917213439941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919773556, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919773557, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784919789143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659141421318054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919804577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311493068933487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919820099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710561394691467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919835351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544154167175293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919850852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322908103466034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919865699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287001222372055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919880725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280333250761032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919880726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919880726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919924214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096800923347473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919924215, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919924215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784919939245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003703951835632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919954317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466036319732666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919969779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14213794469833374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919984879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387247920036316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920000059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151821494102478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920015017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335967481136322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920030069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359827220439911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920030069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920030070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920073817, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114066004753113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920073817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920073818, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784920088767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459280133247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920103890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363254338502884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920119248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552916049957275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920134468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328468799591064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920149564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417071104049683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920164920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296806812286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920180082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250118494033813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920180083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920180084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920224869, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132560729980469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920224870, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920224870, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784920239919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131813183426857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920255144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13452652096748352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920270523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14166438579559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920286118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382403552532196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920301504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416588306427002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920316635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323540359735489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920331889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319306075572968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920331889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920331890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920376238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5153238773345947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920376238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920376239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784920391323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382841020822525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920406452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297414779663086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920421485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12876644730567932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920436698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996092438697815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920452180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799883425235748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920467691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315605640411377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920482609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625012338161469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920482610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920482610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920528036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.517638087272644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920528037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920528037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784920543312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368926465511322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920558476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364012062549591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920573915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144710659980774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920589089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010232150554657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920604530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152003288269043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920620163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526633381843567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920635804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277718424797058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920635805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920635805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920679566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5202612280845642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920679567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920679567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784920694876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588733971118927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920710403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229835033416748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920726454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378944218158722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920742128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626648485660553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920757898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854160904884338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920773453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344437599182129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920789401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13912193477153778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920789401, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920789402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920833525, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5228205323219299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920833526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920833526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784920849506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499140739440918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920865323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1463238000869751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920881232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512015342712402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920897138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289035856723785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920912978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413697481155396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920928611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298919796943665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920944402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366630643606186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920944402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784920944403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784920989017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5259344577789307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784920989018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784920989018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784921004826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610176742076874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921020499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523831963539124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921036181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378581404685974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921051646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360755145549774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921067225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316300511360168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921082666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296989619731903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921097945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255813091993332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921097945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921097946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921142728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5298064947128296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921142729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921142729, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784921158005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260619342327118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921173596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258787989616394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921189325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510340452194214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921204728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352960467338562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921220166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657385110855103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921235685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287747025489807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921251234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997108697891235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921251235, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921251235, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921295305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5352206230163574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921295306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921295306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784921310525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245373964309692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921325740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12707102298736572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921341034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331184059381485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921356419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132249653339386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921372053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699487805366516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921387856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218563795089722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921403014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413512706756592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921403015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921403015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921447885, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5422940850257874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921447886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921447886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784921463468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12942102551460266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921479053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036374747753143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921494331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306404173374176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921509870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12966175377368927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921525137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12779854238033295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921540548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12313179671764374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921556329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269644945859909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921556330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921556330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921599470, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5521054267883301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921599471, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921599471, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784921615147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583536446094513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921630771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317465782165527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921645857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13913577795028687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921661356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357873260974884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921676846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777772545814514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921692177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297505646944046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921707683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269921064376831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921707683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921707684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921752455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5643366575241089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921752455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921752456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784921767498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12909889221191406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921782627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273937076330185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921798104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259825110435486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921813668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275998830795288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921829305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148801028728485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921844507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337626576423645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921860033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037747144699097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921860034, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921860034, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921903507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5791488289833069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921903508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921903508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784921919373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977653741836548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921935227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977471947669983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921951151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298542022705078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921967141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259246945381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921982890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363569974899292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921998481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295236349105835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922014811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304255723953247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922014812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922014812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922057620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5928864479064941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922057621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922057621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784922073436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852585315704346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922089515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111618161201477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922104809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852555513381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922120598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12791217863559723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922136644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733086943626404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922152511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182595372200012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922168536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240772485733032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922168536, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922168537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922211858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6069369912147522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922211859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922211859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784922227516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995675206184387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922243433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279309093952179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922259658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304291933774948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922275842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198265433311462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922291943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313673436641693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922307635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912610173225403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922323886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921777367591858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922323887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922323888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922367644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6194417476654053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922367645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922367645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784922383483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956127524375916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922399606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774479389190674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922415818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893599271774292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922431780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352531909942627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922447961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053521513938904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922464275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600654363632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922480728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12965598702430725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922480729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922480729, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922524684, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6304243803024292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922524685, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922524686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784922540375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12602877616882324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922556515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12810704112052917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922572316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726643681526184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922588049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704221904277802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922603723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1250847578048706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922619495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12222418189048767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922635251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12352356314659119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922635252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922635253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922678324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6415353417396545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922678325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922678325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784922694603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12664708495140076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922710687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228942796587944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922726639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663462966680527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922742301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12276381254196167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922758113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11807802319526672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922773696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11864574998617172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922789319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11661645025014877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922789320, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922789320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922833188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6531572937965393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922833189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922833189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784922849138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198789417743683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922864704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11810622364282608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922880457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439875513315201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922896386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12071357667446136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922912515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11927647888660431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922928734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183721125125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922944498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918429493904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922944499, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784922944499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784922988238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6801076531410217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784922988239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784922988239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784923003472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599129438400269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923019259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682086437940598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923035061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978936940431595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923050674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455747485160828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923066388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332206428050995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923081984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144975945353508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923097641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018805205821991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923097688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923097689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923141450, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.701538622379303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923141451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923141451, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784923156926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157451644539833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923173041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134406104683876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923188974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11801420897245407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923205082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341647058725357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923220512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136995255947113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923236288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114256620407104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923251797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798509418964386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923251797, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923251798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923297091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7277140021324158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923297092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923297092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784923312669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636245250701904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923328253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485715955495834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923344071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233215034008026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923359800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175940930843353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923375443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591794550418854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923390927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977137625217438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923407089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11571056395769119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923407090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923407090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923451583, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7456414103507996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923451583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923451584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784923467299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769203841686249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923483374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500446498394012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923498717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11808063089847565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923514316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384100675582886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923530135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528225988149643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923546050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100723147392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923561929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088373214006424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923561930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923561930, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923605575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7565746307373047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923605576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923605576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784923621239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069820821285248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923636918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013772338628769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923652476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508379876613617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923668277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024945974349976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923684360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469034105539322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923700342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467571765184402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923716382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11360634863376617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923716383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923716383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923759640, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7613509297370911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923759643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923759643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784923775632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846826434135437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923791162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078247427940369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923807190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186320126056671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923823392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12242364883422852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923839291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170804500579834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923855292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196745932102203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923871373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11680937558412552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923871373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923871373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923914406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7648982405662537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923914407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923914407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784923930325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644043028354645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923946441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899905860424042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923962613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11859843134880066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923978668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11186729371547699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923995002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435550451278687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924010742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438188701868057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924026651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474897176027298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924026651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924026652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924071441, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669612765312195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924071442, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924071442, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784924087669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860683560371399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924103718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795393586158752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924119905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590022802352905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924136186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406229436397552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924152413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11871536076068878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924168642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732609033584595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924185253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145951896905899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924185253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924185254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924229646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686635851860046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924229647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924229647, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784924245895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487111449241638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924262485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071426197886467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924278344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861726105213165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924294173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383840441703796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924310005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756292939186096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924325548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407228767871857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924341146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634016036987305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924341147, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924341147, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924385251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699960470199585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924385252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924385252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784924400922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021571606397629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924416564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501757800579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924432228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649479925632477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924447768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528801381587982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924463847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10276180505752563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924479680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959303230047226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924495824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567370057106018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924495825, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924495825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924540567, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770634651184082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924540568, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924540568, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784924556193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526717454195023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924571503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972138494253159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924587659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184321790933609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924603770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986176133155823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924619654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637146234512329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924635293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097648203372955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924651188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583192110061646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924651188, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924651189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924696477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713674902915955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924696478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924696478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784924712225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535275936126709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924727882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572896152734756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924743807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934792459011078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924759783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746745765209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924775849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055745780467987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924791862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118713021278381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924808322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013941466808319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924808323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924808323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924853542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720485329627991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924853542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924853542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784924869105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450772196054459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924884767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203459858894348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924900598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839046537876129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924916487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858448594808578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924932305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473168641328812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924947896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043920576572418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924963671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470272600650787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784924963671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784924963672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784925008923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724165320396423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784925008924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784925008924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784925024481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486114025115967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925040155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954991191625595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925055575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436537861824036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925071512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039449274539948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925087508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604806244373322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925103815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345354676246643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925119904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956728458404541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925119904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925119904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925164544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77279132604599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925164545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925164545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784925179822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09985347092151642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925195874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282643139362335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925211996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722675919532776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925228052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368628799915314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925244006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251426696777344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925259940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633211582899094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925275837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910055041313171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925275838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925275838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925319635, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730819582939148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925319636, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925319636, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784925335238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105671226978302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925350915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030651181936264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925366428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800306499004364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925382255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596199333667755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925398214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967637598514557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925414270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762940347194672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925430185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104862242937088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925430186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925430186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925475756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733757495880127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925475757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925475757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784925491328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679773986339569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925507153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103645920753479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925522952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454273968935013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925538768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203014314174652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925554515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862123966217041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925570248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311658471822739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925586001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817113518714905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925586002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925586002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925631483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737013697624207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925631484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925631484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784925647224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389487445354462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925663025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107322573661804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925678800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773565620183945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925694764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976738691329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925710760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146238446235657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925726797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228717118501663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925743237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520979762077332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925743238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925743238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925787222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773806631565094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925787223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925787223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784925803209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808892548084259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925818763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964059293270111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925835184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11591681838035583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925851835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539584398269653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925868209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465522646903992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925884499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036117672920227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925901025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615789890289307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925901025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925901025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925945481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773769736289978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925945481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925945482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784925961818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144965887069702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925978067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369216442108154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925994475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004241555929184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926011295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024743318557739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926028026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437264084815979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926044682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214926093816757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926061584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304925382137299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926061585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926061585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926105671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739965915679932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926105672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926105672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784926121731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090010553598404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926138013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070762351155281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926154343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688206553459167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926171027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444613337516785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926187370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821532249450684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926203763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993129014968872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926220538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537560820579529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926220539, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784926220539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed864725970.log b/recommendation_v4/rcp_logs/gbs_16384/seed864725970.log new file mode 100644 index 000000000..5e4f2b344 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed864725970.log @@ -0,0 +1,670 @@ +:::MLLOG {"namespace": "", "time_ms": 1784941176105, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784941176105, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784941197141, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "seed", "value": 864725970, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784941197144, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784941197151, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784941197151, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784942114713, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784942114714, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784942115049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784942249871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1397874653339386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942263588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13952040672302246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942277174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909995555877686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942290951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875611126422882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942304850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874804973602295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942318548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387932002544403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942332253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906052708625793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942332254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942332254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942377030, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49946990609169006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942377030, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942377030, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942390494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13936111330986023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942404195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13923616707324982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942418076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909265398979187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942431908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911877572536469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942445643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902181386947632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942459607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391967535018921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942473821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389692723751068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942473821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942473821, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942518875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4996529519557953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942518875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942518875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942532606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896194100379944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942546859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891905546188354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942560717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893800973892212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942575158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876038789749146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942588873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868606090545654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942603202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875794410705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942617157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846749067306519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942617158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942617158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942660175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998840093612671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942660175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942660176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942674126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138668030500412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942688643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868308067321777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942702585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820159435272217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942716746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842512667179108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942731454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828986883163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942745431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795867562294006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942759960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801729679107666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942759960, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942759961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942803129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001634359359741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942803130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942803130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942816804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377868801355362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942830772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805770874023438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942845123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803532719612122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942859021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774780929088593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942872923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368810385465622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942887024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375179886817932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942901323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368090659379959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942901324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942901324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942944324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005132555961609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942944325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942944325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784942958104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735714554786682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942971999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687273859977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942986256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780754804611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943000578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715724647045135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943014559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590708374977112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943028435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362048089504242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943042395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757574558258057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943042395, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943042396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943084982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009409785270691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943084982, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943084982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943099421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708442449569702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943113541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624493777751923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943127349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136351078748703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943141371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550615310668945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943155570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552266359329224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943170205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352074295282364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943184504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353975236415863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943184505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943184505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943227783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014675855636597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943227784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943227784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943242122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474789261817932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943256528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657715916633606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943271247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380220502614975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943285421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455989956855774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943299745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362118124961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943313988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534332811832428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943328730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686725497245789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943328731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943328732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943371561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502174437046051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943371561, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943371561, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943385804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652342557907104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943400138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682539761066437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943414583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506728410720825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943429069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081860542297363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943443644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329200565814972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943457693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387849926948547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943471704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349833607673645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943471751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943471751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943515198, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029847025871277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943515198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943515198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784943529386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514147698879242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943543574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980298697948456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943558487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158684968948364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943572324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367282271385193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943586282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253670930862427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943600190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166362047195435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943614628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269534707069397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943614628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943614629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943657634, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038523077964783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943657634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943657634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943671349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198304176330566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943686488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421162962913513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943700345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840977311134338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943714521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994785606861115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943728920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339220106601715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943743551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12848405539989471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943757715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297726333141327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943757715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943757716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943800769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049543380737305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943800769, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943800769, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943815640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469362258911133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943829579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331290900707245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943843828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447295129299164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943858049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134947270154953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943872704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603749871253967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943886982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13344573974609375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943901522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682377338409424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943901523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943901523, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943944256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062965154647827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943944257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943944257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943959064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442906737327576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943974006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051185011863708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943988803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340462863445282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944003985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14034810662269592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944018872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13048279285430908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944033914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243676722049713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944048531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138229638338089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944048532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944048532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944091153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5075454711914062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944091153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944091153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944105827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12685006856918335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944120365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313363015651703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944135241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708214461803436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944149813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538818061351776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944165053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234210014343262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944179418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884622812271118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944194043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12797924876213074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944194044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944194044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944237106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5088125467300415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944237107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944237107, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944251486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011112809181213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944266037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348382532596588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944280674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14216548204421997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944295047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436931371688843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944310145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13143469393253326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944324524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368099927902222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944339092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611334562301636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944339093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944339093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944382860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5103601813316345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944382861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944382861, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944397100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476720452308655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944411467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13648277521133423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944426001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564112782478333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944440646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321977853775024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944455298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341719627380371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944470175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306090235710144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944484784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257813453674316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944484785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944484785, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944529091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5119689702987671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944529092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944529092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944543598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199666142463684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944558117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447722792625427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944572877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14176645874977112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944587855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443154096603394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944602666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13449326157569885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944617516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245677947998047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944632075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306507468223572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944632076, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944632076, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944674572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5139074325561523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944674574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944674574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944688712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380854994058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944702974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331152319908142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944717097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12892237305641174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944731291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299445629119873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944745818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381344199180603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944760459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163211941719055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944775110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631653785705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944775110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944775111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944819593, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5159714818000793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944819594, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944819594, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784944834358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13688647747039795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944848812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644367456436157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944863519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131405770778656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944877940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015654683113098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944892662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153545558452606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944907674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350896656513214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944922748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278242945671082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944922748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944922749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944966706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182803869247437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944966706, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944966706, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784944981281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610392808914185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944996361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227123022079468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945011142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137957364320755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945025918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636842370033264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945040671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855871558189392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945055259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445442914962769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945070299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906404376029968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945070300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945070300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945113695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5206590890884399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945113695, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945113695, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945128692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508188724517822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945143669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15216492116451263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945158903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515588641166687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945173888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330583542585373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945188958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342441439628601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945203803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323432207107544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945219014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660134375095367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945219015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945219015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945261976, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5233317017555237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945261977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945261977, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784945277071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620872795581818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945292175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534657657146454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945307289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381671905517578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945321703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371899724006653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945335979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337525725364685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945350328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298483908176422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945364594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12567779421806335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945364595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945364595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945407717, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5265664458274841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945407718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945407718, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784945422079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271771371364594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945436752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325855255126953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945451483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506320118904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945466179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542675971984863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945481270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683456182479858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945496077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290503889322281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945510742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299334615468979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945510743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945510743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945554566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5313462018966675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945554567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945554567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784945569085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254761695861816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945583583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722261250019073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945598169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328896462917328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945612938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246473670005798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945628159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712794542312622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945643585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246652483940125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945658299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440266251564026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945658300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945658300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945702454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5377703905105591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945702455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945702455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784945717595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294432282447815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945732474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061505556106567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945747210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085895776748657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945761907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996526062488556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945776389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282065063714981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945791213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12349121272563934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945806386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12756773829460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945806386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945806387, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945849317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5464287996292114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945849318, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945849318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784945864425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599476218223572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945879402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367964327335358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945893917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139394611120224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945909032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391107320785522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945923741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809649109840393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945938251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005471229553223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945952989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12729567289352417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945952989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784945952990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784945997183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5576443076133728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784945997185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784945997185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784946011636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935388088226318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946026067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280178725719452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946040910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303247094154358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946055768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315528631210327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946070727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318482607603073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946085287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411089777946472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946100411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099901378154755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946100411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946100412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946144075, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5729410648345947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946144076, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946144076, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784946158952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999296188354492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946173782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042987883090973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946188806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060805201530457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946204013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317570090293884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946219087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443955779075623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946233875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303776204586029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946249461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311403512954712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946249461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946249462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946292392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5878069400787354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946292392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946292393, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784946307691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889480590820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946323112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190972805023193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946337740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941960990428925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946353190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12869682908058167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946368362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282958984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946383529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326577365398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946398899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338316977024078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946398900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946398901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946443432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6023815870285034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946443433, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946443433, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784946458291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079842925071716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946473628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12925681471824646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946489337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114801049232483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946505037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326189935207367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946520584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419604301452637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946535683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043132424354553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946551491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949973344802856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946551491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946551492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946594310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6154360771179199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946594311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946594311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784946609555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063737750053406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946624942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12896248698234558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946640336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304023265838623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946655369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624638319015503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946670773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317741572856903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946686295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12728184461593628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946701771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105323910713196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946701771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946701772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946746551, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6267677545547485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946746552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946746552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784946761575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278369277715683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946777121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288319230079651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946792206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852126359939575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946807188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288486272096634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946821949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670984864234924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946836812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1242666244506836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946851723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12474069744348526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946851724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946851724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946894373, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6390237212181091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946894374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946894374, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784946909836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285734921693802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946925171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12447385489940643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946940268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11835350841283798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946955438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12438046932220459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946970538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030699104070663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946985470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12096985429525375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947000390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878373473882675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947000391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947000392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947044413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6516324877738953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947044414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947044414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784947059511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384901195764542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947074322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11928658187389374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947089145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676369607448578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947104022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321335822343826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947119130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12112273275852203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947134447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120366171002388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947149362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124363541603088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947149363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947149363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947194483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6766093373298645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947194483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947194483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784947209176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769430339336395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947224344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11768792569637299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947239051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12129522860050201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947253477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636258661746979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947268009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479775607585907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947282319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161797046661377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947296792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213655769824982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947296838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947296839, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947341068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6977601647377014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947341069, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947341069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784947355363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168668195605278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947370434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481854319572449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947385139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11954870820045471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947400398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503467708826065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947414970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470947414636612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947430240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1220119446516037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947445047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943911969661713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947445047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947445048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947488012, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7180039882659912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947488012, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947488012, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784947502951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763903498649597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947517963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577755212783813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947532977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283320188522339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947547659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247989535331726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947562478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727857053279877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947576957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975014209747314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947592383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161745935678482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947592383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947592383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947636446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7344949841499329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947636446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947636446, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784947651355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878872662782669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947666780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587142944335938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947681797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11850404739379883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947697173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494295507669449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947711979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546510457992554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947726646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109782755374908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947741370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944682359695435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947741371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947741371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947784709, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7452856302261353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947784710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947784710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784947799122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11168606579303741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947813597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097590625286102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947827922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544740200042725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947842443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061068624258041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947857364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155254989862442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947872208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494926571846008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947887256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392386257648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947887256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947887257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947930973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7531366348266602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947930974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947930974, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784947945687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954242944717407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947960110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087527871131897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947975875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12154411524534225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947991427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12230253219604492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948006572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694938689470291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948021856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223800480365753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948037193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721522361040115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948037194, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948037194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948082163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7579126358032227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948082164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948082164, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784948097239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610789597034454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948112565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898416489362717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948128049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11817934364080429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948143410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117287129163742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948159135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411139369010925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948174388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452365666627884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948189814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053793653845787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948189815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948189815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948236179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7611420750617981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948236180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948236180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784948251278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11905965209007263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948266455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774779319763184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948281753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605702549219131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948296852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390349268913269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948311876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11863984167575836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948327061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734884440898895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948342882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421683430671692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948342883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948342883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948387365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640437483787537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948387366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948387366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784948402805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507615447044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948418523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754940658807755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948433548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859168320894241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948449350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11360616236925125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948464536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078757792711258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948479324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417751967906952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948494136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701902747154236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948494137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948494138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948538297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7653827667236328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948538298, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948538298, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784948553174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063875257968903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948567958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497624963521957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948582707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668044537305832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948597456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534407943487167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948612675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265389084815979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948627737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09651325643062592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948643005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608121752738953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948643006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948643006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948687722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670707106590271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948687723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948687723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784948702609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105240598320961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948717326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985193401575089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948733066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872665584087372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948748364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098187267780304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948763321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660284757614136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948778012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125099867582321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948793066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621906816959381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948793067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948793067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948837027, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686648964881897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948837028, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948837029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784948851815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537247359752655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948866251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559126734733582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948880873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960381478071213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948895626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776357352733612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948910564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640833526849747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948925488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187651753425598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948940779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048026382923126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948940780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948940780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948984744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696915864944458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948984745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948984745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784948999481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521098971366882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949014970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023985892534256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949030089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891799628734589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949045278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085682213306427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949060278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495427250862122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949075196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011374294757843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949090185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490946471691132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949090186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949090186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949134455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704461216926575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949134455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949134456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784949149252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484594851732254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949164220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002215206623077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949179004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10514163225889206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949194266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105571836233139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062052920460701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949224958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382558405399323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949240346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981971025466919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949240346, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949240347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949284379, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713139653205872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949284380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949284381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784949299025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000424087047577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949314197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280404984951019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949329077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736552625894547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949343936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345062613487244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949358730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224951803684235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949373445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701732337474823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949388280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905104875564575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949388281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949388281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949431823, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716182470321655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949431824, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949431825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784949446478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584117472171783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949461133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399790108203888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949475669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831552743911743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949490581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614696145057678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949505656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11015608161687851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949520765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073777824640274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949535835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093337833881378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949535835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949535836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949579882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719883918762207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949579883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949579884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784949594522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704845190048218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949610209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187636315822601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949625350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459263622760773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949640510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018192321062088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949655506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865829885005951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949670484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375370621681213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949685298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837974399328232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949685299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949685299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949728425, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724793553352356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949728425, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949728425, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784949743420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362036526203156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949758483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117930710315704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949773427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797378420829773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949788550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993715584278107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949803718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231495440006256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949818900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128617525100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949834708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554447025060654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949834709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949834709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949877771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729527354240417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949877771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949877771, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784949892956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082712858915329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949907683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993724524974823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949923528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590301990509033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949939050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11566458642482758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949954349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476646363735199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949969635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383887588977814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949985150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652823746204376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949985150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784949985151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784950028162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729090452194214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784950028163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784950028163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784950043607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114928126335144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950058671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375338584184647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950074041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011599749326706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950089625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038783192634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950105277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451004445552826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950120699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241459846496582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950136462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268934607505798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950136463, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950136463, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950180442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733930945396423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950180443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950180443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784950195534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864710807800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950211316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744385421276093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950226708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752913355827332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed869903124.log b/recommendation_v4/rcp_logs/gbs_16384/seed869903124.log new file mode 100644 index 000000000..2cda15e72 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed869903124.log @@ -0,0 +1,800 @@ +:::MLLOG {"namespace": "", "time_ms": 1784878421935, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784878421936, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784878442425, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "seed", "value": 869903124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784878442427, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784878442459, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784878442460, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784879337356, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784879337358, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784879337684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784879483719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902723789215088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879498004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388220191001892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879511997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388496607542038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879526129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869400322437286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879540300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869109749794006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879554490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386108249425888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784879568537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870951533317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879568538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879568538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879613004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49948135018348694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879613004, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879613005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784879627154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871479034423828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879641067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388287991285324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879655272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862305879592896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879669543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385851800441742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879683825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870257139205933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879698290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838618993759155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879713012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813239336013794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879713013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879713014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879757729, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4996693432331085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879757729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879757730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784879772233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857746124267578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879786808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383836269378662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879800895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838255405426025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879815849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813820481300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879830279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833756744861603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879845381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784964382648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879859731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819152116775513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879859731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879859732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879903825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49989718198776245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879903826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879903826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784879918120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850408792495728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879933407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797779381275177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879948159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760192692279816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879962969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843271136283875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879978330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783353567123413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784879992728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771922886371613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880007750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751110434532166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880007750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880007751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880052683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001887679100037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880052684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880052684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784880067215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371850073337555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880081863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778796792030334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880096909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757671415805817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880111332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13713082671165466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880125857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360544115304947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880140307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371259093284607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880154992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614502549171448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880154992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880154993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880199250, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005394816398621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880199250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880199251, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784880213873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692352175712585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880228520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652163743972778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880243286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375226378440857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880258065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367616057395935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880272435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523969054222107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880286734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555759191513062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880301297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741439580917358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880301298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880301298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880346063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009147524833679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880346063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880346063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784880361072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368352472782135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880376005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584862649440765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880390513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359136998653412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880405190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513784110546112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880419989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352650225162506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880435318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473102450370789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880450237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351621001958847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880450238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880450238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880494196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013654232025146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880494197, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880494197, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784880509199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426682353019714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880524174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365453004837036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880539581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380923092365265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880554574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421359658241272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880569784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360633373260498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880584555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498283922672272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880599780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652828335762024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880599781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880599782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880644403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501996636390686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880644403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880644404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880659449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364193558692932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880674649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672363758087158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880689663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480229675769806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880704539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046202063560486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784880719561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326901614665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880734330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379594683647156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880749027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503769040107727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880749073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880749073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880792899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027307868003845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880792900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880792900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784880807724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523536920547485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880822415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294761449098587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880838021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150382041931152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880852535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368393450975418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880867221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229474425315857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880881820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13134543597698212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880896893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255684077739716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880896893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880896894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880941105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503585159778595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880941106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880941106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784880955458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179387152194977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880970802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424454629421234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880985398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12805727124214172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881000039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978601455688477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881014755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375438749790192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881029760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12843844294548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881044288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991228699684143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881044288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881044289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881088837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047359466552734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881088838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881088838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784881104607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13452830910682678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881119235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304005563259125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881134431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441838324069977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881149592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495436310768127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881165107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360294669866562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881180008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335815191268921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881195122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696858286857605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881195123, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881195124, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881239389, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062566995620728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881239389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881239389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784881254514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438254594802856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881269745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060498237609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881284976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397793471813202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881300374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1402631402015686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881315534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036981225013733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881331110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324351727962494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881346676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828888535499573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881346677, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881346677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881391479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077824592590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881391480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881391480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784881407116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666308879852295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881422587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311078816652298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881438139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696922361850739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881453485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548436760902405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881468879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323229968547821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881483542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890875339508057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881498423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280996948480606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881498424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881498424, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881542411, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5093724131584167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881542412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881542412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784881557198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300535947084427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881572025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485436141490936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881587203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14211055636405945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881602043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133957639336586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881617841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314096301794052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881632798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379377126693726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881647926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605107367038727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881647926, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881647927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881691714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112033486366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881691714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881691714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784881706623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471680879592896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881721617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633757829666138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881736926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567891716957092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881752049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328413665294647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881767436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430365920066833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881782572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285613059997559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881797431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253292441368103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881797432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881797432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881841596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5131807923316956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881841597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881841597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784881856315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187536597251892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881871236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345425546169281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881886380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14173981547355652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881901735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338273286819458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881916855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342877745628357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881932139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323838233947754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881947398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330711543560028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881947398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784881947399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784881991671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5155107975006104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784881991672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784881991672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784882006775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810886442661285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882021903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329961121082306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882036978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879669666290283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882052119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000085949897766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882067562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800230622291565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882082922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156217336654663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882098003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630540668964386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882098004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882098004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882142121, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5181513428688049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882142122, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882142122, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784882157195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369515061378479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882172002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636893033981323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882187079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316046267747879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882201916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008445501327515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882216973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153241574764252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882232210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539931178092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882247467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270655274391174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882247468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882247468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882291335, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5213719606399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882291336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882291336, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784882306132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594774901866913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882321686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243332505226135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882337470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783761858940125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882352729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363075077533722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882368338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833120465278625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882383682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429144024848938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882399424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13901996612548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882399424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882399425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882444104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248342752456665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882444104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882444104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784882459703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507553935050964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882475310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15003672242164612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882491014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350673884153366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882506306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297200202941895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882521676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413770496845245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882536856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331249177455902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882552359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652577996253967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882552359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882552359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882597022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5288404822349548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882597022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882597023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784882612517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362319141626358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882628004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352708786725998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882643637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374681770801544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882658962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358551263809204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882674215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313594460487366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882689301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954360246658325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882704299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254587322473526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882704299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882704300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882748579, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5335767865180969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882748579, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882748579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784882763564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325266808271408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882778862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323854774236679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882794176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499264419078827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882809390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351875364780426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882824709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365991234779358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882839821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287517547607422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882855025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981286644935608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882855026, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882855026, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882899726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5407536625862122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882899726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882899726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784882914622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132473424077034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882929581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693768739700317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882944574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300460577011108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882959511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205119967460632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882974898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12701192498207092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882990482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204741477966309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883005411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391397893428802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883005412, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883005412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883049971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5507375001907349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883049971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883049972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784883065638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293145716190338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883080688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031598925590515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883095748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043394684791565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883110917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970122694969177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883125916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12773172557353973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883141213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299135327339172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883156882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705975770950317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883156882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883156883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883201026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5633708834648132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883201027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883201027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784883216588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547179102897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883232055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329178363084793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883246955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852578401565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883262542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292622566223145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883277735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734726071357727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883293010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916061282157898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883308305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12633724510669708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883308306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883308306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883353814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5779109597206116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883353815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883353815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784883368812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861087918281555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883383780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12696023285388947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883399325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185226917266846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883415016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210925459861755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883430696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081499934196472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883445845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323232531547546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883461543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987250089645386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883461544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883461545, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883506116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5923579335212708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883506116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883506116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784883521453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907058000564575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883536845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889514863491058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883552165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12865829467773438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883567392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13161945343017578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883582387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320609450340271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883597313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852944433689117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883612992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293717324733734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883612992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883612992, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883655595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6036451458930969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883655596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883655596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784883670795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726420164108276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883686414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979458272457123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883701372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704327702522278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883717290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12657874822616577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883733275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12619024515151978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883749103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000261783599854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883765141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308237910270691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883765141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883765142, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883807965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6143229603767395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883807966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883807966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784883823314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811371684074402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883839143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12624892592430115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883855231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890909612178802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883871270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306958794593811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883887172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210442662239075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883902685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12727555632591248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883919039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12766143679618835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883919039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883919040, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883962634, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6242018938064575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883962635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883962635, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784883978264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12818753719329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883994044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259239763021469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884009732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692531943321228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884025164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346745073795319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884040978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931716442108154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884056859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12428872287273407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884072771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670081853866577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884072772, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884072772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884117025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6322973370552063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884117025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884117025, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784884133220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341340631246567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884150170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262967735528946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884166001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12494393438100815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884181437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12448546290397644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884196728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12288210541009903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884212236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963628232479095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884227726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.122336745262146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884227727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884227727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884270982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6421008706092834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884270983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884270983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784884287026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12490452080965042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884302877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12152063101530075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884318684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140320897102356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884334697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208811104297638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884350262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634863913059235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884365668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11660975217819214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884381027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424744129180908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884381028, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884381028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884424704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6541092395782471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884424705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884424705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784884440342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034326791763306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884455498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608635634183884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884470699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114271953701973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884486109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11954939365386963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884501908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830929666757584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884517765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11737175285816193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884533178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850750654935837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884533179, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884533179, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884577534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6686692833900452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884577534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884577534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784884592577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603310704231262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884608757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159333884716034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884624487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189829558134079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884639808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367775499820709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884655233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201710999011993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884670562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359912902116776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884685969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030973494052887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884686017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884686017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884730798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.686305820941925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884730799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884730799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784884745923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484992504119873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884761790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325390636920929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884777353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11897536367177963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884793255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301518976688385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884808396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422395706176758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884824467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995816230773926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884839913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813666880130768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884839913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884839914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884884582, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7019391059875488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884884582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884884582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784884900036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674494296312332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884915397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472740024328232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884931047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219489574432373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884946403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210930347442627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884961621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648125410079956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884976691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201966404914856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884992457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633791029453278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884992458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784884992458, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885036798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7186846733093262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885036799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885036799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885052376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887113749980927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885068318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452525109052658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885083730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11837680637836456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885099388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459060966968536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885114779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11651688814163208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885130279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11167623102664948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885145712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946372151374817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885145713, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885145713, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885189923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7306598424911499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885189924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885189924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885205122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218371987342834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885220508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125865578651428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885235721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636927723884583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885251233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230327934026718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885266917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638173460960388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885282562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106814906001091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885298420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531634628772736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885298421, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885298421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885341574, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7388095259666443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885341574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885341575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885357226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982125252485275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885372410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270520836114883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885388680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12351125478744507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885404533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12614545226097107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885420088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12052823603153229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885435800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347337067127228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885451498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11888405680656433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885451499, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885451499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885495833, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7411333322525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885495834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885495834, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885511429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185310110449791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885527298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132147908210754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885543449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12059813737869263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885559464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352504789829254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885575666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11809920519590378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885591448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577446758747101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885607630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710237920284271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885607631, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885607631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885651043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7431813478469849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885651044, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885651044, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784885666898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998830735683441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885682753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978075861930847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885698600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635251551866531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885714320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11613015830516815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885730024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983199417591095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885745843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090974509716034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885762049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564761400222778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885762050, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885762050, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885806598, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7473428249359131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885806599, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885806599, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784885822381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663596332073212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885838553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109881691634655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885854029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005009710788727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885870310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156681627035141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885886160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877615958452225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885901730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051831990480423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885917150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11642705649137497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885917151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885917151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885961727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7491852045059204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885961728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885961728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784885977248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150628328323364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885992765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527034640312195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886008275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948403179645538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886023674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618782043457031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886039544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369804501533508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886055377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09754347801208496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886071452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063532829284668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886071453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886071453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886116010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7507283687591553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886116011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886116011, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886131601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609328001737595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886146942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000120639801025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886163026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983487755060196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886178649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948535054922104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886194060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070617288351059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886209276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168589651584625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886224780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107625812292099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886224780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886224781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886269713, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7523094415664673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886269714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886269714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886285282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520683228969574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886300619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639762878417969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886316157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030781269073486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886331711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793404281139374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886347474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654143989086151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886363311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236825585365295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886379342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047788709402084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886379342, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886379343, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886424057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7528225779533386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886424058, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886424058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886439512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627776384353638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886455371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025695651769638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886470964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096490919589996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886486799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968382656574249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886502371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509342700242996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886518006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995420277118683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886533740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486344993114471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886533741, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886533741, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886578447, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7533925771713257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886578448, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886578448, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886594022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490253567695618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886609701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107666254043579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886625161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521187633275986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886641120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067203432321548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886657033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675753653049469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886673230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400919616222382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886689102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030931770801544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886689103, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886689103, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886734263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7548802495002747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886734264, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886734264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886749443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999969601631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886765667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393793135881424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886781335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747300088405609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886796817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454963147640228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886812252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274485498666763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886827688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828486829996109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886843198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917554795742035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886843199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886843199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886888443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.755776584148407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886888443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886888444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886903889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637389868497849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886919343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329841822385788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886934748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920552909374237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886950443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662408918142319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886966228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031889170408249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886982206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829818993806839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886998057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123397946357727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886998058, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886998058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784887042873, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7583844065666199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784887042874, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784887042874, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784887058182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688069462776184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887074232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142569243907928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887089906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526770353317261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887105642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241837799549103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887121295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863736271858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887136918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323200911283493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887152628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897528380155563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887152629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887152629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887197967, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7578659653663635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887197968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887197968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784887213762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561420768499374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887229488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148138344287872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887245253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805236548185349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887261181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011404752731323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887277271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318198591470718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887293352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344785243272781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887309837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635009407997131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887309837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887309838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887355417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597779631614685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887355418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887355418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784887371240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875151306390762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887386789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980037808418274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887403583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622019857168198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887419796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11738927662372589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887435793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509484052658081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887451735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419213771820068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887467723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698643326759338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887467724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887467724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887513506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7594877481460571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887513506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887513506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887529503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188510060310364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887545420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472668498754501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887561500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107189953327179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887577794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073224991559982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887594005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500691622495651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887610102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303189396858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887626342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349945515394211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887626343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887626343, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887670665, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606602311134338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887670666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887670666, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887686211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007742583751678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887702838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765440762042999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887719194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719819366931915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887735675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508440971374512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887751952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11806926131248474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887768258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040113866329193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887784782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563509702682495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887784782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887784783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887830744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7615730166435242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887830745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887830745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887846837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409120261669159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887863143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383748054504395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887879412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626587808132172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887895816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362634599208832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887912269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929984599351883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887928506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975191742181778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887944506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239226162433624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887944507, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887944507, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887989699, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619365453720093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887989700, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887989700, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784888005684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952140390872955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888021684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542389750480652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888037197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526394844055176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888052850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467644035816193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888068375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697239637374878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888084344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385341942310333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888100199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905382037162781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888100199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888100200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888145098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640102505683899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888145098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888145099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784888160830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999567806720734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888176368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322894901037216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888192322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850055515766144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888208148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371177643537521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888224238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278913378715515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888240431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591438412666321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888256147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963692724704742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888256147, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888256148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888301247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7643990516662598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888301248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888301248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784888316598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828794538974762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888333319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073216795921326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888349601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654619336128235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888365639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616447031497955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888381393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804960131645203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888397026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034943163394928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888412771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855419933795929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888412771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888412771, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888457644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659206390380859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888457645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888457645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784888473546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825395584106445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888489197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587707161903381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888505214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856816172599792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888521150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351316630840302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888537299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960492491722107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888553422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959599167108536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888569749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490000247955322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888569750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888569751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888615088, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7663514614105225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888615088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888615088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784888631389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841026902198792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888647076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360360145568848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888663330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340406537055969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888678593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369633883237839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888694727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066264867782593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888710540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027437150478363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888726125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619745403528214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888726125, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888726126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888770694, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671012282371521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888770694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888770694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784888786124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334204137325287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888802080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084840655326843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888818139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931108891963959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888834261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066321283578873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888849993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471417009830475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888866024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977821797132492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888882269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982029139995575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888882270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888882271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888926921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680212259292603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888926922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888926922, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784888943104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174552142620087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888959546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516181588172913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888975560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046343743801117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888991086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675226151943207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889007216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143038630485535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889023283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797788202762604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889039665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049303188920021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889039665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889039665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889084013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7684004902839661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889084013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889084014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784889100002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764630138874054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889115693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868345946073532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889131445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069389641284943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889147262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026312410831451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889163096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741430729627609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889179072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261938720941544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889194847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366559773683548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889194848, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889194848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889239946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678852677345276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889239947, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889239947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784889256089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315199196338654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889272026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913345962762833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889288075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921309888362885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889304341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111593097448349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889320469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843849927186966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889336150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074727028608322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889352646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026947945356369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889352647, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889352648, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889398312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692918181419373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889398313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889398313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784889414321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948358476161957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed871175675.log b/recommendation_v4/rcp_logs/gbs_16384/seed871175675.log new file mode 100644 index 000000000..0ff1bc641 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed871175675.log @@ -0,0 +1,661 @@ +:::MLLOG {"namespace": "", "time_ms": 1784931508107, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784931508107, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784931528935, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "seed", "value": 871175675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784931528936, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784931528937, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784931528937, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784931528937, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784931528939, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784931528940, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784932438963, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784932438964, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784932439276, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784932580766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902899622917175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932594521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881325721740723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932608139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387811303138733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932622210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873691856861115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932636274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387946456670761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932650229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857662677764893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784932664221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884514570236206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932664221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932664222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932709788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4968470633029938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932709789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932709789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784932723519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868482410907745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932737424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386837363243103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932751402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874012231826782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932765370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860547542572021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932779449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867872953414917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932793726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384655237197876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932808118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836073875427246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932808120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932808120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932851396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49698713421821594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932851396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932851396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784932865486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384759247303009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932879885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833951950073242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932893745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834773004055023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932908413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381358504295349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932922389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832315802574158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932937056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766631484031677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932951326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800638914108276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932951326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932951327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932994400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4971446394920349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932994400, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932994400, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784933008420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853543996810913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933023172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807103037834167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933037427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744056224822998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933051770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843324780464172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933066667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768184185028076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933080942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756471872329712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933095767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750487565994263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933095768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933095768, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933139271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49734166264533997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933139272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933139272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784933153127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370665580034256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933167219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770446181297302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933181583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752980530261993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933195938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701370358467102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933210267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569214940071106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933224721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369839310646057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933239150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598297536373138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933239150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933239151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933282397, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49756094813346863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933282397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933282398, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784933296335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678669929504395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933310174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631349802017212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933324387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754230737686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933338864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665705919265747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933353099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497915863990784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933367454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548313081264496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933381843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727280497550964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933381844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933381844, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933425438, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49781399965286255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933425438, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933425438, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784933440070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369035392999649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933454486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577337563037872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933468541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581126928329468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933482805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494572043418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933497131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512113690376282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933512074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459348678588867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933526552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13491909205913544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933526553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933526553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933569194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4981750249862671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933569195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933569195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933583664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416744768619537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933598229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363629251718521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933613585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818703591823578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933627984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387508690357208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933642693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614973425865173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933657180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348433494567871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933671967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366879940032959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933671968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933671968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933715289, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49874094128608704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933715290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933715291, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933730010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653454184532166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933744659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367771029472351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933759253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346583366394043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933773864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301383674144745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933788697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247182965278625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933802978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352563977241516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933817038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496796786785126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933817085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933817085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933860286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4994160234928131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933860287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933860287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933874664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500413298606873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933888841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291210800409317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933903783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131157785654068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933917924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687366247177124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933932209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323564201593399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933946281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314118653535843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933960813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324690878391266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933960814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784933960814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934003108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002778768539429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934003108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934003109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934017012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316462755203247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934031963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418924808502197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934045954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783677875995636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934060029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296645700931549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934074356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373741507530212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934088930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282173991203308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934103066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983134388923645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934103066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934103067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934147579, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014894008636475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934147580, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934147580, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934162429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466744124889374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934176506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305309414863586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934190822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442564010620117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934205069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502296805381775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934219815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618305325508118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934234225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352656364440918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934248962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368906944990158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934248962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934248963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934293207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029221177101135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934293207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934293207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934308189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419315218925476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934323155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055714964866638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934337866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381865620613098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934352898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14053812623023987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934367765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053297996520996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934383013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243404030799866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934397675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381128430366516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934397676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934397677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934441718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042513012886047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934441719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934441719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934456653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12407394498586655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934471279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13123857975006104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934486137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723178207874298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934500828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541635870933533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934516187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238279521465302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934530547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893913686275482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934545087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12804856896400452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934545088, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934545088, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934588737, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5056440234184265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934588738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934588738, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784934603203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000616431236267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934617602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348503679037094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934632356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14225783944129944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934646832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134354367852211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934661993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13143879175186157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934676516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336519569158554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934691065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617655634880066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934691066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934691066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934735677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072787404060364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934735678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934735678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784934749975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134632870554924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934764471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634544610977173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934779240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566583395004272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934793723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322316110134125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934808447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437673449516296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934822981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329214721918106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934837447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323450803756714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934837447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934837448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934881644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5089629292488098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934881645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934881645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784934896032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188812136650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934910457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444176316261292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934924971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14176449179649353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934939814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341807246208191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934954536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426759839057922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934969414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224947452545166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934984065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332228183746338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934984066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784934984066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935027015, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5109360814094543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935027015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935027015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935041465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895739614963531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935056102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302703201770782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935070569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288870871067047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935085179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008329272270203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935099886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807561993598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935114777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177338242530823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935129451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640940189361572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935129452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935129453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935173469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.513274610042572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935173470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935173470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784935188216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367449313402176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935202764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632291555404663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935217616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13136141002178192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935232141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000023365020752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935246841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154500722885132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935261628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515014946460724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935276576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254210352897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935276577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935276577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935320708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5158866047859192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935320708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935320708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784935335299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585209846496582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935350437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227733969688416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935365335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792279362678528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935380111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643261790275574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935395029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859879970550537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935409869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436071574687958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935425109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390305757522583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935425110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935425110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935469458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5184902548789978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935469459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935469459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784935484637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350354701280594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935499813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.15146097540855408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935515493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504885137081146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935530680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330661177635193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935545909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417887687683105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935560870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311198353767395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935576115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647237420082092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935576115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935576116, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935618733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5217211842536926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935618734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935618734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784935633913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360282003879547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935648947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352231204509735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935664278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376736640930176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935678911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367503881454468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935693517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314656913280487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935708092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928389012813568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935722544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256500631570816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935722544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935722545, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935764581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.525676429271698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935764582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935764582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784935779125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266190886497498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935793947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259001076221466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935808910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348743736743927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935823754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539229333400726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935838597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659392297267914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935853293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879693508148193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935867699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983573973178864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935867699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935867700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935910241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.531480610370636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935910242, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935910242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784935924636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250958919525146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935938941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270725429058075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935953315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309501111507416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935967824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321968287229538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935982822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269868016242981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935997984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221295177936554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936012451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13412758708000183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936012451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936012452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936055993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5391553640365601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936055994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936055994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784936071181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944775819778442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936085971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037657737731934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936100613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065500557422638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936115335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12973718345165253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936130057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276702582836151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936144823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12326689809560776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936160028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722575664520264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936160029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936160029, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936202978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5498698949813843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936202979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936202979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784936218087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551470637321472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936233070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315194845199585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936247623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873447477817535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936262849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321971893310547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936277849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12778958678245544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936292774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294868290424347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936307992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12678344547748566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936307992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936307993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936351424, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5627915263175964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936351425, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936351425, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784936366146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288830041885376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936381021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721475958824158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936396362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230851292610168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936411584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132390558719635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936426776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13115954399108887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936441729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133550226688385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936457060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020730018615723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936457061, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936457062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936499633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5773030519485474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936499634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936499634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784936514639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935934960842133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936529509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961652874946594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936544396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12919726967811584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936559258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219064474105835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936573958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328771412372589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936588535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291702389717102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936603777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988005578517914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936603778, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936603778, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936647274, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5895342230796814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936647275, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936647275, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784936662203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277206838130951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936677368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023991882801056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936691936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12754595279693604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936707215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12725073099136353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936722470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267867386341095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936737689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094647228717804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936752953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157255947589874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936752953, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936752954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936797534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6002740263938904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936797535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936797535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784936812358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12923932075500488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936827536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273198127746582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936843089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12953850626945496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936858656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159576058387756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936874088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324934959411621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936889044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808498740196228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936904948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12863972783088684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936904949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936904949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936947671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6087287664413452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936947672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936947672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784936962757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128921777009964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936977893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706276774406433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936993013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12788823246955872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937007902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13439533114433289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937023149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001874089241028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937038474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248738020658493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937053883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750157713890076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937053884, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937053884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937098422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6149845123291016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937098423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937098423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784937113425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12440183758735657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937129361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714490294456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937144287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12580382823944092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937159237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584654986858368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937174037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12401483952999115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937188939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12103386223316193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937203816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311579287052155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937203817, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937203817, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937246543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.62086421251297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937246544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937246544, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784937261951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12573757767677307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937277230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12202973663806915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937292425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152971163392067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937307753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169694900512695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937322790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709412187337875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937337521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765670776367188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937352294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11533275246620178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937352295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937352295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937395017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.627290666103363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937395018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937395018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784937410021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12098344415426254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937424842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11706917732954025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937439562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260509490966797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937454446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12092092633247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937469591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11928682029247284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937484877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11806206405162811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937499783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917207598686218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937499784, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937499784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937543623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6357048153877258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937543624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937543624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784937558260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11672553420066833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937573921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701032519340515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937589152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11974721401929855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937604160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451993882656097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937619242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131005585193634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937634122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145680621266365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937649204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108524352312088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937649251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937649251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937692409, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6454594731330872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937692410, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937692410, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784937707136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593388020992279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937722591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378683149814606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937737907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965464055538177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937753561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394363641738892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937768416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483985930681229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937783676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12066234648227692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937798451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093069463968277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937798452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937798452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937841744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6573528051376343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937841745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937841745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784937856459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776351928710938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937871325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531385779380798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937886462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134580671787262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937901352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232081800699234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937916262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803982615470886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937931036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12145970016717911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937946472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717969179153442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937946473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784937946473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784937989957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6732457876205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784937989957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784937989958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784938005247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12015033513307571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938020802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587564647197723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938035762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11932182312011719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938051199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556544363498688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938066235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168302595615387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938081293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252988874912262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938096358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087489873170853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938096359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938096359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938141123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6929145455360413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938141124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938141124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784938156100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307196319103241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938171077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187836527824402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938185867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11729243397712708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938200931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327570676803589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938216187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11740513145923615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938231442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836631059646606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938246796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597743630409241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938246796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938246797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938291561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7115158438682556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938291562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938291562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784938306781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061210930347443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938321706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318899691104889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938337556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239330917596817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938353013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126948744058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938368280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12213870882987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938383654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446896940469742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938398948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12013712525367737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938398949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938398949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938442734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7280189394950867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938442735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938442735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784938457801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021504342556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938473124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256362497806549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938488692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12279421091079712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938504137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463887989521027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938519869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11928413808345795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938535149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714315414428711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938550913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898706316947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938550914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938550914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938595862, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7402312755584717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938595863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938595863, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784938611523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1205558255314827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938627249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196719110012054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938642990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793234407901764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938658672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11804059892892838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938674211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12145721912384033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938689921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138493567705154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938706130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779290437698364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938706131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938706131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938749832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7494733333587646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938749833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938749833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784938765531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11879274994134903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938781489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201075464487076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938796662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172729730606079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938812189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167951226234436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938827171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052420735359192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938841971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637503862380981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938856657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11653681099414825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938856657, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938856658, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938900249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7553513646125793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938900250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938900250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784938915038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291614919900894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938929873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627399921417236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938944739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11194784194231033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938959424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836437344551086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938974639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062232255935669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938989824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943190217018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939005212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854524374008179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939005212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939005213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939051279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.759407103061676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939051279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939051279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784939066231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752908885478973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939080934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109967529773712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939096733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12195451557636261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939111925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017598956823349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939126901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819390416145325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939141759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313780605792999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939156905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905063152313232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939156906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939156906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939201116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.762498676776886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939201117, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939201117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784939216165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610402375459671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939230947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768327862024307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939246128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106085404753685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939261350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958807170391083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939276703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816256701946259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939292083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303859949111938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939307629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122960597276688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939307630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939307630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939352825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7644369006156921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939352825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939352826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784939367798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644248127937317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939383340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315839946269989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939398542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153413355350494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939413867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100628674030304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939428922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061631590127945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939443879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043776780366898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939458930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503624379634857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939458930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939458931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939504173, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652004361152649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939504174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939504174, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784939519165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527049005031586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939534315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144820809364319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939549308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594604164361954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939564769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108587682247162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939580191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657070577144623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939595879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380889475345612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939611398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052601784467697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939611398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939611398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939656769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7668716907501221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939656770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939656770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784939671537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029365122318268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939687185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378184169530869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939702201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794523358345032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939717354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423065721988678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939732325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238784551620483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939747325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805638134479523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939762442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109676294028759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939762442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939762443, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939805709, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671761512756348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939805710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939805711, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784939820759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675735771656036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939835718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309891402721405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939850444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969308763742447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939865678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655219852924347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939880938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067094653844833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939896343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839985311031342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939911553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130528151988983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939911554, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939911554, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939955149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687733769416809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939955149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939955150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784939969933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065317839384079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939985662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103076696395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940000881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556922852993011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940016143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219375789165497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940031189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860024392604828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940046183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332394182682037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940061238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903492569923401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940061239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940061239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940104754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688633799552917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940104755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940104755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784940119811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525277256965637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940135027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130712926387787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940150147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794790089130402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940165430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980109333992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940180823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112694650888443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940196173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258135735988617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940211942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543863475322723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940211942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940211943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940256363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695468068122864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940256364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940256364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784940271549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846488177776337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940286492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100139319896698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940302540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11606051027774811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940318062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667993664741516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940333352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489252001047134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940348574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441732406616211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940363885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691270977258682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940363886, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940363886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940408055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695039510726929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940408056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940408056, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784940423379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205874383449554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940438482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430022120475769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940453804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033763736486435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940469355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081426590681076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940484818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145239770412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940500267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234401166439056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed925729918.log b/recommendation_v4/rcp_logs/gbs_16384/seed925729918.log new file mode 100644 index 000000000..e441a7c4a --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed925729918.log @@ -0,0 +1,672 @@ +:::MLLOG {"namespace": "", "time_ms": 1784950494619, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784950494619, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784950515588, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "seed", "value": 925729918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784950515592, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784950515593, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784950515668, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784950515668, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784951368593, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784951368594, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784951368898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784951573283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847559690475464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951587247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859286904335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951600834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862386345863342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951614700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859890401363373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951628630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386515498161316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951642311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872584700584412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951655958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831403851509094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951655958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951655959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951699419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002884864807129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951699421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951699421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951713037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828764855861664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951726830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838374614715576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951740743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840734958648682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951754643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840578496456146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951768631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383359134197235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951782784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809296488761902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951797180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844585418701172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951797180, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951797181, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951839932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004202127456665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951839932, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951839933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951853968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838502764701843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951868163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381653994321823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951881851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798266649246216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951896411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806968927383423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951910321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800083100795746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951924795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811883330345154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951938642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379132866859436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951938642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951938643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951981158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005621314048767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951981158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951981158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951994972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831275701522827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952009572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378287971019745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952023795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719812035560608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952038073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825343549251556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952052727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766539096832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952066593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372944712638855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952081024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371997594833374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952081025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952081025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952124473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007615685462952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952124475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952124475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784952138172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685496151447296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952152063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755913078784943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952166426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734744489192963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952180189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693445920944214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952193906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568298518657684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952207878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677969574928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952222038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585889339447021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952222038, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952222039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952264916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010018348693848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952264916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952264916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952278905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681122660636902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952292866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627581298351288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952307066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374949961900711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952321398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365034580230713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952335110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350172609090805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952348824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560235500335693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952362743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738888502120972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952362743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952362743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952406715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012800693511963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952406716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952406716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952420968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367274820804596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952435017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566577434539795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952448994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588997721672058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952463077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490422070026398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952477200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506457209587097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952491758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451695442199707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952505886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349780261516571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952505887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952505887, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952549558, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016886591911316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952549559, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952549560, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952563742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13412126898765564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952578081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640056550502777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952592857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809014856815338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952606994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339997947216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952621310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13589414954185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952635445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489577174186707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952649998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660871982574463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952649999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952649999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952693119, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022938251495361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952693119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952693120, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952707384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645346462726593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952721712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662844896316528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952736051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480007648468018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952750412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023634254932404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952764645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234871625900269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952778629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337299346923828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952792534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480296730995178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952792578, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952792579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952836636, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029956698417664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952836638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952836638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952850879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510215282440186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952865020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12932640314102173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952879769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111761212348938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952893624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683368265628815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952907568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233676552772522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952921443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13136358559131622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952935801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266521692276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952935802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952935802, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952979086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037782788276672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952979086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952979086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952992807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163766264915466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953007769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432399928569794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953021813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12795770168304443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953035822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954191863536835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953050100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337074190378189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953064537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12821783125400543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953078527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296144276857376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953078528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953078528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953123114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047837495803833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953123115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953123115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784953137951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460713624954224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953151904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332055628299713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953166055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447433710098267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953180125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347612738609314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953194680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624250888824463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953208876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335715353488922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953223423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369268149137497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953223424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953223425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953267224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060900449752808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953267224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953267224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784953281709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341484934091568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953296359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130398228764534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953310921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384300470352173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953325776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14038455486297607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953340377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304011195898056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953355301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323011815547943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953369696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803283870220184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953369697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953369697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953411141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5073729157447815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953411143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953411143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784953425862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263532191514969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953440538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100352883338928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953455601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693298399448395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953470341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354098618030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953485442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233479857444763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953499546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877774238586426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953513865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12794113159179688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953513865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953513866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953555522, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086999535560608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953555522, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953555523, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784953569773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018035888671875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953584142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471965491771698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953598842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1420166790485382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953613221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411740958690643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953628402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144239783287048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953642602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337738335132599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953657072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359492540359497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953657072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953657073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953698629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5102251172065735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953698629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953698629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784953712681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347733587026596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953727055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363220363855362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953741667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557443022727966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953756310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331101357936859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953770948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425880670547485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953785175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292795419692993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953799226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322193443775177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953799227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953799228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953841809, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118816494941711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953841809, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953841810, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784953855798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200098276138306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953869725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428696990013123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953883872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141882985830307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953898229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373717665672302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953912669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418996334075928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953927297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225239515304565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953941902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307349383831024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953941903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953941903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953983417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138294100761414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953983417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953983418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784953997741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873139023780823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954012157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295164704322815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954026401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287592202425003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954040625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986283004283905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954055207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791675865650177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954069851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165168464183807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954090242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361825168132782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954090243, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954090249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954133086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160717368125916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954133087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954133087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784954147333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681581616401672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954161326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363932490348816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954175632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127340376377106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954189672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997892498970032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954203951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151150941848755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954218565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351618468761444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954233095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263463973999023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954233095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954233095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954276257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5186848044395447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954276257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954276258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784954290551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593986630439758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954305757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220757246017456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954320781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790008425712585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954335634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621971011161804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954350566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385124772787094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954365206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343858540058136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954380414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894164562225342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954380415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954380416, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954422897, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5212972164154053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954422898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954422898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784954438109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498806953430176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954453333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1490492820739746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954468550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497553765773773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954483354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289058208465576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954498306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418936729431152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954512936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323372602462769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954527821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638247549533844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954527821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954527822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954570771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5244552493095398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954570772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954570772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784954585620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621337711811066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954600515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523226976394653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954620610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373342156410217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954635357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354156911373138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954650143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309341669082642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954664803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952929735183716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954679163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560254335403442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954679164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954679164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954721240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5284707546234131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954721241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954721241, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784954735750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325659155845642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954750616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244441151618958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954765537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350260078907013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954780327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354188472032547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954795240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658851385116577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954809974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12865783274173737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954824600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980452179908752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954824601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954824601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954867883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5346848368644714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954867883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954867883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784954882202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325284242630005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954896696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269327998161316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954911316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303467631340027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954925963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214609026908875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954941050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268824338912964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954956099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212671875953674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954970644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341215968132019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954970645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784954970645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784955015843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5431119799613953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784955015844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784955015844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784955031153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941491603851318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955045873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030022382736206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955060362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050046563148499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955075034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969328463077545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955089418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12767964601516724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955104043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12320388853549957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955118981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704813480377197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955118982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955118982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955163216, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5548246502876282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955163217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955163217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784955178000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557949662208557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955192745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288652896881104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955207142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860586285591125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955222465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331670731306076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955237370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747061252593994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955252050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12940847873687744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955266939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266246736049652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955266940, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955266940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955309520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5686448812484741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955309521, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955309521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784955324053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288244128227234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955338621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12710420787334442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955353682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218986988067627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955368727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324102282524109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955383967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116693496704102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955398744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325512409210205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955413933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006539642810822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955413933, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955413934, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955457906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5833144187927246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955457906, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955457906, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784955473013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289735585451126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955488025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944483757019043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955503140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289118230342865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955518229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192719221115112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955533082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247233629226685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955547702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871405482292175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955562891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971818447113037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955562891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955562892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955606252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5955064296722412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955606253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955606253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784955621295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12732404470443726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955636679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13014715909957886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955651426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12727046012878418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955666830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12690164148807526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955681926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12656167149543762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955697002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045816123485565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955712335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111813366413116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955712336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955712336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955755850, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6056730151176453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955755851, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955755851, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784955770583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880948185920715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955785830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265765130519867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955801258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922745943069458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955816579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13146041333675385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955831926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224807381629944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955853864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277250051498413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955869675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12817862629890442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955869676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955869676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955912918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6147444248199463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955912919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955912919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784955928019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282023787498474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955943207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604525685310364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955958465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270100474357605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955973454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339838206768036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955988692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960472702980042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956004204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12424337863922119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956019779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267642378807068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956019779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956019779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956063463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6218673586845398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956063463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956063464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784956078456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12387694418430328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956093738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686672806739807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956108465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252410113811493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956123204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12427061796188354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956137896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296458333730698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956152592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11964499950408936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956167318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12239247560501099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956167318, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956167318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956209955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6297379732131958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956209956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956209956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784956225141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12507547438144684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956240260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12105543166399002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956255214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421870440244675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956270415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12104573845863342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956285534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631909012794495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956300328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11723732948303223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956315095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428992450237274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956315096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956315096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956357517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6405841112136841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956357518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956357518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784956372600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021856009960175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956387378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11612571775913239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956402245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163477599620819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956417283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114871293306351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956432636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11864139139652252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956448077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11774095892906189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956463040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979671776294708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956463040, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956463041, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956507092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6524690985679626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956507093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956507093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784956521770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670201271772385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956536886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11585171520709991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956551533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11933852732181549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956566105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389288306236267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956580643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269092559814453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956595077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418379843235016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956609599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063864827156067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956609644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956609644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956653313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6700317859649658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956653314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956653314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784956667564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576317250728607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956682375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296428740024567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956697050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12011946737766266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956712175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283627897500992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956726727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465032398700714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956742064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11989132314920425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956756944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872720181941986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956756944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956756945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956800783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6912520527839661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956800783, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956800783, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784956815509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708204448223114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956830220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443966627120972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956845082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248376220464706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956859890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182686686515808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956874761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690146684646606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956889417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12044252455234528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956904678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1171296238899231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956904678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956904678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956948647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7089563012123108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956948648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956948648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784956963635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039895355701447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956978841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505624651908875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956998837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887788772583008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957013939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057792529463768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957028578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172284185886383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957043369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119542345404625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957058268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959914326667786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957058269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957058269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957101889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7291717529296875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957101890, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957101890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784957116487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338400840759277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957131347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210057139396667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957145959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701711267232895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957160818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320964992046356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957175875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636252701282501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957190950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723128914833069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957205985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649778485298157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957205986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957205986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957249039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7425472140312195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957249040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957249040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784957264118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085571348667145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957283226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377017199993134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957298832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12386348843574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957314218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269831657409668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957329312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12120848894119263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957344516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491996794939041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957359618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970505118370056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957359619, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957359619, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957402927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7503325939178467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957402928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957402928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784957417872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11937803030014038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957433024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233241856098175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957448377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12216776609420776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957463681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416807770729065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957479201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887681484222412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957494263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168297603726387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957509854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863080620765686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957509855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957509856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957552499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7555879354476929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957552500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957552500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784957567746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049564719200134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957582995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108987033367157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957598429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621588677167892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957613681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755560338497162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957628947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12148471176624298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957644298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050166189670563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957659996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11735906451940536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957659996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957659997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957702691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7609198093414307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957702692, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957702692, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784957717909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11748023331165314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957733575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127479374408722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957748815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096462607383728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957767826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630222201347351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957782787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009502410888672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957797438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058194488286972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957811976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11588724702596664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957811977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957811977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957854068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637112736701965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957854068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957854068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784957868875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116243153810501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957883768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618204414844513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957898542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202012002468109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957913311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667037963867188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957928645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475722700357437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957943711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09893938153982162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957959044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843673348426819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957959045, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784957959045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784958000688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661231160163879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784958000689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784958000689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784958015590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692629218101501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958030237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090894043445587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958046337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208338662981987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958061883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099386215209961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958077137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728442668914795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958092304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027291864156723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958107714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831934958696365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958107715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958107715, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958150090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673848867416382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958150090, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958150091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784958165497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542112588882446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958180712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735294222831726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958196130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029212176799774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958211640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858043283224106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958227255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771728307008743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958242969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305209994316101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958258905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152664572000504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958258906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958258906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958301916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679041624069214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958301917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958301917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784958317365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630768537521362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958335692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298795998096466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958350823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126118898391724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958366056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992397367954254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958380912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544025897979736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958395738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037203133106232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958410788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049066036939621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958410789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958410789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958453108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678709626197815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958453108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958453109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784958467946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459521412849426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958482712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007406935095787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958497290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548203438520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958512519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100980639457703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958527768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655676573514938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958543191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387039929628372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958558532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077690124511719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958558532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958558532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958601063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690240740776062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958601063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958601064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784958615704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979033470153809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958631562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421855002641678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958646964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763227939605713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958662228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498203337192535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958677387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223498195409775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958692514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759517550468445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958707838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969722270965576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958707838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958707838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958750179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687154412269592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958750180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958750180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784958765432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676732659339905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958780680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297736525535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958795763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956752300262451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958811325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071600615978241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958826942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036671698093414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958842563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815511643886566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958858050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138460040092468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958858051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958858051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958900695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7700597643852234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958900695, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958900696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784958915596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620564222335815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958931206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121849924325943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958946123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522536933422089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958961188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255203396081924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958975995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880730301141739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958990755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287356913089752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959005669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087692379951477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959005670, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959005670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959048076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694944739341736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959048077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959048077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784959063028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513302683830261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959078065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127573251724243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959093142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836528986692429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959108572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006607860326767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959124067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277516186237335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959139483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134573370218277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959155449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054871678352356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959155449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959155449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959200113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703933715820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959200114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959200114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784959215202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844822227954865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959230112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019782930612564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959246364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11582972854375839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959262381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164027601480484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959277799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531390994787216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959293229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043301373720169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959308793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686278343200684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959308793, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959308794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959353124, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697263360023499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959353125, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959353125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784959368713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182036995887756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959384137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401896178722382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959399746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060595512390137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959415516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062347143888474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959431330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459290981292725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959446802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228188127279282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959462610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314369738101959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959462610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959462611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959507395, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707025408744812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959507395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959507395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784959522553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966705530881882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959538644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748502612113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959554087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705696791410446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959569714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453980207443237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959584923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11833789199590683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_16384/seed951784539.log b/recommendation_v4/rcp_logs/gbs_16384/seed951784539.log new file mode 100644 index 000000000..05f13ca5b --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_16384/seed951784539.log @@ -0,0 +1,813 @@ +:::MLLOG {"namespace": "", "time_ms": 1784851975239, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784851975239, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784851996535, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 16384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "seed", "value": 951784539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 2e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784851996537, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784851996538, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784852868526, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784852868528, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784852868845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784853035087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13977870345115662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.5833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853049455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395249366760254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2499999999999997e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853062967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908663392066956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853076590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882756233215332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.583333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853090469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385909914970398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.25e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853104404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887521624565125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.916666666666666e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784853118092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906452059745789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853118093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853118094, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853162033, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49850887060165405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853162034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853162034, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784853175564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394306719303131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853189398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392485499382019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853203226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389772742986679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853217038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914638757705688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853230848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890136778354645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9916666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853245135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390654444694519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853259629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873830437660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853259630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853259631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853302818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49868327379226685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853302818, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853302819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784853316660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895556330680847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853331042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389259696006775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853344925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892263174057007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8249999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853359637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877283036708832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.9916666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853373443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860848546028137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853387886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830067217350006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853402000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847076892852783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853402000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853402001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853444966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989106059074402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853444966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853444966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784853459069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868898153305054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6583333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853473824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865353167057037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.8249999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853488102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824690878391266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853502472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836650550365448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853517515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829484581947327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3249999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853531732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817493617534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853546417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382242739200592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853546418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853546419, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853590202, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49920740723609924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853590203, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853590203, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784853604196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783115148544312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853618286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381637006998062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853632821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790035247802734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1583333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853646943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742251694202423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853661019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672737777233124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.4916666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853675165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746941089630127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853689581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136905699968338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853689582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853689583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853733757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49959617853164673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853733757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853733757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784853747706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727951049804688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853761543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690972328186035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853775770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783463835716248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853790389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696962594985962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853804326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579711318016052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.658333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853818338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604602217674255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853832445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376153826713562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853832446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853832446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853876659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000578165054321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853876660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853876660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784853891130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708563148975372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853905500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611625134944916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853919365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363174021244049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.491666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853933612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553360104560852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853947958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560551404953003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853962952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517211377620697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853977677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533194363117218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.158333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784853977678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853977678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784854021214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006815195083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784854021214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784854021214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784854035707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462725281715393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.325e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854050073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365996152162552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854065034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813352584838867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854079761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450849056243896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.824999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854094623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362687051296234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.991666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854109199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522803783416748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.158333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854124188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368955671787262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.324999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854124189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854124190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854168217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014812350273132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854168217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854168217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784854182780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367320716381073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.491666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854197414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13676312565803528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.658333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854212031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509845733642578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.825e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854226591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307719349861145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.991666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784854241255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328062117099762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854255702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396000862121582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854270050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507068157196045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854270102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854270102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854313806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023983716964722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854313806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854313806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784854328384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504526019096375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854342726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12966354191303253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854357594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131600022315979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854371742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671740889549255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.1158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854385944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244110345840454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854400069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168656826019287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854414701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325618326663971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854414702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854414703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854457862, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034013390541077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854457863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854457863, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784854471739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200458884239197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854486500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413919508457184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854500457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281592696905136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854514476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299247443675995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854528732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391238451004028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854543262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284601092338562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854557275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988990545272827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854557276, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854557276, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854600655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046472549438477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854600655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854600655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784854615653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456977903842926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854629704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297349214553833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.3158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854644012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446299731731415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854658161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502052426338196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854672858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595229387283325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.3658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854687161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355115056037903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854701813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702735304832458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854701814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854701815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854746096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060774087905884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854746096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854746097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784854760551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440021872520447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.4158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854775117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305731236934662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854789620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395574688911438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.4491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854804423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14039865136146545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.4658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854819024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042327761650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854834286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256768882274628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854849325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827170431613922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854849326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854849326, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854894325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5073773860931396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854894326, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854894326, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784854909403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254481077194214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854924245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310766339302063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854939436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699129223823547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854954400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353837251663208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854969570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230520486831665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854983962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287952959537506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.6158333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854998494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812493741512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854998494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784854998495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784855041995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5087918639183044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784855041995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784855041996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784855056404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301097422838211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855070983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347505748271942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.6658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855085747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14208923280239105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6825000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855100209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415130972862244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855115548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314110904932022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.7158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855129989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367614150047302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855144811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601581752300262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855144812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855144812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855188322, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5104959011077881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855188323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855188323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784855202883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471533358097076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855217607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625389337539673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855232680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563483953475952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855247575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322389125823975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855262584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440245389938354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855277499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132987380027771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855292001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132482647895813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8658333333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855292001, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855292002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855335946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122043490409851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855335946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855335946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784855350430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183578848838806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855365081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448020815849304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8991666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855379812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141771137714386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855394798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338716447353363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9324999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855409630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434725999832153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855424479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227760791778564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855439199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322272896766663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855439200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855439201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855482021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5142053961753845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855482021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855482021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784855496470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831019401550293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855510967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330241858959198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855525385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289616823196411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855539892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004809617996216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855554636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791781663894653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855569483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156160712242126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855584322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620129227638245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855584323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855584323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855627749, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5165714621543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855627750, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855627750, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784855642404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672053813934326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.1158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855656887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642993569374084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1324999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855671574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151815533638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1491666666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855685952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017573952674866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1658333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855700639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156163692474365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1824999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855715584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549986481666565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855730556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266444206237793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2158333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855730556, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855730557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855773325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5192568898200989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855773326, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855773326, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784855787904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601410388946533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855803186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232021033763885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855818801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378486156463623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855834082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645252585411072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855849580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854020833969116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855865038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343478560447693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855880936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914990425109863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855880936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855880937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855923824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5220048427581787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855923825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855923825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784855939360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351572573184967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855954833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1474159061908722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855970259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516643643379211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3824999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784855985166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133060023188591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856000270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423460721969604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856015052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313522934913635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856030080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662061095237732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856030080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856030081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856073148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5253992080688477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856073149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856073149, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784856088182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610520958900452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856103297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518142700195312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856118806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376283645629883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856133926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370278477668762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856149152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133277028799057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856164284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938231229782104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856179294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255214512348175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856179295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856179295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856221845, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5294685959815979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856221845, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856221845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784856236775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234540820121765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.5824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856252069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259878754615784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856267412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349216252565384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856282583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536718487739563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856297668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367337703704834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856312644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287517249584198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856327625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298348754644394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856327626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856327626, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856371199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5351846814155579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856371199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856371200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784856385750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323830485343933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.699166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856400386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12709780037403107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856414977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323813676834106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856429654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321672797203064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856444846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706798315048218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856460189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13223931193351746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856474829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421255350112915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856474830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856474830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856517893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5429272055625916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856517894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856517894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784856533413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294001042842865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.8158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856548340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042163848876953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856563107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052666187286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856577945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976936995983124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.8658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856592475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12781012058258057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856607123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12334531545639038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856622050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721571326255798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856622051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856622051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856665170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5531517863273621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856665171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856665171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784856680125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572606444358826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856694914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321761786937714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856709353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898617029190063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856724620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361455500125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856739480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276819109916687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856754287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964972853660583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856769303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126911461353302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856769304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856769304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856811980, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5657010078430176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856811981, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856811981, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784856826629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128914475440979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856841414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12761792540550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856856482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255253434181213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856871610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265672326087952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856886797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145321607589722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856901700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368065655231476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856917043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304277777671814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856917044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856917044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856959515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5795171856880188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856959516, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856959516, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784856974473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296239048242569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784856989253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982077896595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857004055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12977895140647888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857018897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326110064983368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857033490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359391689300537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857047726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963911890983582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857062735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033534586429596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857062735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857062736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857106128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5917503833770752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857106129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857106129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784857120814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829571962356567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857135891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092881441116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857150335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12839409708976746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.3158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857166104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278197169303894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857181756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747280299663544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857197354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194936513900757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857212901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244834542274475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857212901, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857212902, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857256065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6049648523330688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857256066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857256066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784857271295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300986260175705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857286977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281396448612213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.4158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857302902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13028928637504578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857318719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320466697216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857334476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347317278385162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857349843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922336161136627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857365820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12906329333782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.4991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857365821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857365821, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857408851, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6199292540550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857408852, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857408852, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784857423861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296074241399765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.5158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857439150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278473436832428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857454495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291283369064331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.5491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857469595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352851837873459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857484918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077189028263092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857500571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259632706642151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857516189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958770990371704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857516190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857516190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857559945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6332611441612244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857559945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857559945, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784857574896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606002390384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6325000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857590585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280054748058319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857605885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711311876773834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857621067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12715399265289307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857636108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252662092447281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6991666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857651164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12256310880184174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857666220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12359119951725006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857666221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857666221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857710623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6493738889694214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857710624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857710624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784857726213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264951080083847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857741520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12323768436908722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857756714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673526465892792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.7824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857772101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12266702950000763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857787291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11833084374666214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857802262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889462918043137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8324999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857817244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1166396513581276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.8491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857817244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857817244, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857862159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6687652468681335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857862160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857862160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784857877203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12178266048431396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857891964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1180567592382431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857906708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430901288986206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.8991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857921599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12064269930124283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.9158333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857936765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11912749707698822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857952083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11843380331993103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.9491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857967003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911882668733597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.9658333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784857967004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784857967004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784858010952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7190183997154236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784858010952, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784858010952, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784858025508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568094789981842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858040947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622567474842072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858055997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925844103097916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.0158333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858070738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135980412364006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858085485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292877793312073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.0491666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858100198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391014605760574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0658333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858114932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956189036369324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0824999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858114990, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858114990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858158927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7450937032699585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858158928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858158928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784858173372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398570239543915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858188557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217360198497772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.1158333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858203620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174994707107544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858219045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183597147464752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.1491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858233886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189030110836029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1658333333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858249123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923982203006744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858263891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621768236160278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858263891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858263892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858309052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7599461674690247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858309053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858309053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784858324006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437100172042847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858338881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275321990251541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858353866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042416095733643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.2491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858368613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979124903678894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858383369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403941571712494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858397957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682511866092682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2991666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858413301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229068040847778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858413302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858413302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858457543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.765038251876831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858457544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858457544, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784858472577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537792533636093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858488006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255918443202972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.3491666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858502921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150406002998352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3658333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858518350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097943246364594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3824999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858533415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377923935651779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.3991666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858548647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654614120721817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.4158333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858563915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540561378002167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4324999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858563916, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858563916, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858608268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676844596862793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858608269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858608269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784858623265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802226513624191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.4491666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858638400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741771757602692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858653358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240646988153458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.4824999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858668477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791085660457611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858683887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152725666761398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858699223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154286026954651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5324999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858714706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096731424331665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858714706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858714707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858758188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694432735443115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858758188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858758188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784858773490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659456253051758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858788464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890606045722961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.5824999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858804206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860384047031403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5991666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858819737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12121735513210297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858835318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349359154701233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6324999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858850929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105959564447403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.6491666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858866545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507739126682281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858866546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858866546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858908855, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706847190856934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858908856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858908856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784858924318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379161477088928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858939920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729272663593292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.6991666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858955760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662890017032623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858971404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002273857593536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784858987411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131453663110733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.7491666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859002877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332535743713379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859019077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382398962974548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859019078, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859019078, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859062380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715259194374084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859062381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859062381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784859077936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11810429394245148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859093425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594377666711807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859108964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463782399892807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859124439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320088803768158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859139845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11753396689891815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859155470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606750845909119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859171492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365388333797455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859171493, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859171493, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859215175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722527384757996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859215175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859215175, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784859230713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480388045310974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859246586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697754472494125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859261856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773228108882904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859277831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292294412851334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859293062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072423905134201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859308116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400329530239105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859323112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11637115478515625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.015833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859323113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859323113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859366255, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723494172096252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859366256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859366256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784859381331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958072543144226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859396435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386846005916595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859411457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594604909420013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859426511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477620363235474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859442000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169149935245514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859457423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09581378102302551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859473070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557590425014496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859473070, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859473071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859517174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731118202209473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859517175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859517175, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784859532340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104611337184906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859547314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004723608493805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859563353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727826297283173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859579043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925933718681335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859594397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554684698581696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859609622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054537653923035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859625139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609760880470276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859625140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859625140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859669313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735000252723694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859669314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859669314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784859684765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479633510112762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859699768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530713200569153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859715059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910331457853317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859730548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666949301958084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859746045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591058433055878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859761494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159217357635498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859777237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894042253494263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859777238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859777238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859822266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773842990398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859822267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859822267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784859837338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044512391090393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859852713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219275951385498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859868171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832234472036362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859883665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885071754455566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859899046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474218428134918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859914290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079826414585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859929706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489971935749054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784859929706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859929707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859974151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740800976753235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859974151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859974152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784859989305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430637747049332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860004438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996665358543396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860019136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418014228343964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860034425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987190902233124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860049917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057700514793396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860065517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324975103139877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860080761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962473601102829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860080761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860080762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860124786, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742171287536621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860124786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860124787, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784860139600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957945346832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.615833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860155481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282144695520401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860170835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675491392612457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860186282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032351702451706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860201762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242350399494171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860217245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586179047822952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860232810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917812585830688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.715833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860232810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860232811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860277259, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745112180709839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860277260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860277260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784860292797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477420687675476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860308330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325432568788528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860323794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758255422115326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860339605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566399991512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860355365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947977006435394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860371270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107231006026268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860386914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108068972826004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860386914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860386915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860430056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748541831970215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860430056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860430057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784860445143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655468702316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860460790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016412228345871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860475957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410970449447632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860491201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194375365972519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860506335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822223126888275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860521386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305699497461319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860536347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796105861663818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860536348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860536348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860580359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747754454612732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860580360, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860580360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784860595588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356120765209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860610585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071601510047913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860625619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734091699123383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860640913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973365068435669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860656224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137358099222183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860671498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232250928878784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860687304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467194020748138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860687305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860687305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860730537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746170163154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860730538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860730538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784860745666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731877386569977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.0825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860760532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969064593315125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860776668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561498045921326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860792597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505136638879776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860808100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444780230522156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860823770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409136116504669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860839481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599321126937866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.1825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860839482, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860839482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860883130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742201089859009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860883131, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860883131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784860898812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098089814186096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860914436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135120540857315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860930427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940209031105042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860946525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994853079319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860962792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141912117600441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860978783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205662786960602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.2825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860995110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293764412403107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784860995111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784860995111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784861038992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747712135314941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784861038993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784861038993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784861054266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090703010559082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861070344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709428787231445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861085959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067553162574768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861101963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423467099666595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861117338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830250918865204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.3825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861132876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937145352363586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861148736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11570192128419876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.415833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861148737, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861148737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861192548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750566601753235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861192549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861192549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784861207860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349136382341385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861223502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365240812301636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861239214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547505855560303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861255092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345110833644867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.4825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861270959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927315056324005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861286544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880571603775024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.515833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861301840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147642135620117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.5325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861301841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861301841, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861345754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752222418785095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861345755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861345755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784861361695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826878249645233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861377361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431358963251114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861392540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359922051429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.5825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861407799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038605272769928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861422974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646893829107285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861438562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256658494472504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861453829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080443263053894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861453829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861453830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861497113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755559682846069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861497114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861497114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784861512286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915820300579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861527246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020842045545578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.6825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861542692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774146020412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861558025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315996408462524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861573553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203651338815689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.7325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861589367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545563697814941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861604710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948772192001343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861604711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861604711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861648987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753822207450867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861648988, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861648988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784861663971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744132846593857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.7825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861679987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009790748357773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861695478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651333630084991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.815833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861710829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058780625462532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.8325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861726014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714523494243622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861740866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315479338169098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861756153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746743530035019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.8825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861756154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861756154, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861799866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756831049919128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861799867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861799867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784861815008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757589340209961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861829972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554182529449463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.915833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861845064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854367911815643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.9325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861860144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244768857955933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861875559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852764546871185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861891059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930828005075455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.9825e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861906709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490131378173828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861906709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861906710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861949699, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755860686302185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861949699, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861949699, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784861965058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747458040714264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.015833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861980003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178473591804504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.0325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784861995577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116437017917633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862010680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262079536914825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862026295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003829747438431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862041513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246284306049347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862056773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531807690858841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.115833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862056774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862056774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862099853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758181095123291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862099853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862099854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784862114964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292338579893112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.1325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862130514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016191005706787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862146278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820866376161575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862161891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933057963848114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862177006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431256890296936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862192446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753269493579865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.215833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862208082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895559936761856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862208083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862208083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862251417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759561538696289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862251417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862251418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784862267117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082680523395538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862282945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414381325244904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862298539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450305044651031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862313659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054667979478836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862329268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996563196182251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.315833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862344784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724349319934845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862360594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401810705661774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862360595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862360595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862403968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760851979255676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862403968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862403968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784862419297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677347332239151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862434484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775505751371384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862449891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969399124383926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862465523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915930569171906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.415833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862481053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09664444625377655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862496552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018233448266983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862511615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378125309944153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862511616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862511616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862555001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761543989181519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862555002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862555002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784862570739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228225588798523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862586274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079125851392746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862601631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108666330575943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.515833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862617217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045290529727936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862632876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076684296131134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862647918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908976197242737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.565833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862663706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176437348127365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862663706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862663707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862708026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767019271850586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862708026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862708026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784862723460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944220423698425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862739380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393519699573517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.615833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862754932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680383443832397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.6325e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862770377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155407130718231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862786112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793647915124893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.665833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862801812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245062947273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862817162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413508862257004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862817163, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862817163, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862861254, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761220932006836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862861255, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862861255, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784862876321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608251392841339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.715833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784862891604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811774641275406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.7325e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed1056795887.log b/recommendation_v4/rcp_logs/gbs_32768/seed1056795887.log new file mode 100644 index 000000000..59653e0f1 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed1056795887.log @@ -0,0 +1,670 @@ +:::MLLOG {"namespace": "", "time_ms": 1784867506376, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784867506376, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784867530027, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784867530030, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784867530030, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784867530030, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1056795887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784867530031, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784867530075, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784867530076, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784868319820, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784868319821, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784868320146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784868583874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398722231388092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868599682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394955962896347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868614697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13954627513885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868622121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868622122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868674016, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49981313943862915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868674017, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868674017, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868681432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13959470391273499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868696362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13964825868606567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868711968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393793225288391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868727358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13929669559001923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868727358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868727358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868750818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.499986857175827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868750819, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868750819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868765873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13973769545555115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868781182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13950994610786438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868796495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13942719995975494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868803890, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868803891, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868827488, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001888275146484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868827488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868827488, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868835404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899779319763184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868850776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906079530715942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868866387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138914555311203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868881826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893060386180878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868881827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868881827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868904824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004494190216064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868904825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868904825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868920044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876968622207642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868935110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387263685464859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868950488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856622576713562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868958079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868958080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868981134, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007497668266296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868981135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868981135, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868988492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828685879707336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869004227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382460594177246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869019178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380765438079834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869034626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379339098930359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869034627, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869034627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869059084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010827779769897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869059085, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869059085, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784869074133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137836754322052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869089532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787750899791718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869105569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725730776786804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869113196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869113197, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869136826, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014649033546448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869136827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869136827, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869144461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772696256637573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869160804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368989199399948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869176286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702628016471863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869192514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735176622867584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869192514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869192515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869216837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019446611404419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869216838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869216838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869232075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684672117233276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869248009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357397437095642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869263182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686548173427582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869270798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869270799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869294861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024238228797913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869294862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869294862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869302473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340218484401703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869318258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512614369392395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869333255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353791058063507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869348269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549679517745972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869348347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869348348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869372748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029433965682983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869372749, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869372749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869387968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342841237783432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869403068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486988842487335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869418171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547426462173462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869426472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869426472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869450035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035143494606018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869450035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869450035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869457412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135753333568573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869472552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511493802070618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869487922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322389841079712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869503916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134807288646698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869503917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869503917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869527355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042659640312195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869527356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869527356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869542695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388673961162567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869558298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351604461669922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869574422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502998650074005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869582311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869582311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869606323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053830146789551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869606324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869606324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869614044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640692830085754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869629755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352081179618835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869646125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343008428812027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869661547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217121362686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869661614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869661615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869685574, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5065687894821167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869685575, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869685575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869700838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283199071884155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869717047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13126058876514435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869732672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330745816230774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869740501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869740502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869764297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5078709125518799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869764297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869764297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869771973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384873241186142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869787507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107624650001526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869803510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986254692077637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869818876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302585005760193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869818877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869818877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869842778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.509249746799469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869842779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869842779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869858292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271939754486084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869873902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976901233196259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869890131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058239221572876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869897852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869897853, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869921726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107477307319641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869921727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869921727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869929530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315629929304123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869945013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508418202400208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869960766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331859529018402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869977082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559654355049133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869977082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869977083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870000908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5123155117034912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870000909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870000909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784870016401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512252271175385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870031942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254289329051971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870047568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332014948129654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870055280, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870055280, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870079625, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138659477233887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870079626, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870079626, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870088318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363433599472046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870104495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517031073570251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870120483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621464371681213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870136732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633191585540771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870136733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870136733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870160838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5153893232345581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870160839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870160839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870177531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472765684127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870193575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079649209976196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870209505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626107573509216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870217672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870217672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870241509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5170062780380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870241509, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870241509, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870249413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586583733558655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870265971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246802985668182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870281688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325032114982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870297358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080202043056488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870297358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870297359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870320875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5187673568725586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870320875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870320875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870336680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097792863845825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870353118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830988109111786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870368920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935273349285126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870376733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870376734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870399877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5207361578941345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870399877, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870399878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870407609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931860983371735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870423315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469098508358002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870439034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344577670097351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870455745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033756613731384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870455746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870455746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870479872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5230439901351929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870479873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870479873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870495555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280662715435028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870511352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318562626838684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870527224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357206404209137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870535292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870535293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870559553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5256800055503845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870559553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870559553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870567549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651970028877258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870584011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334550529718399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870599895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187682628631592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870615478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307375431060791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870615479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870615479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870639434, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5287814140319824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870639435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870639435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870655238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960055470466614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870671041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238534331321716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870687499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527284562587738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870695637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870695638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870718978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5322568416595459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870718978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870718978, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870726906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13174831867218018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870743075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332702934741974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870759024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486693799495697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870775198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432256877422333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870775198, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870775198, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870799354, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5364710092544556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870799355, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870799355, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870815595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576388359069824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870832258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337500810623169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870848388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330413818359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870856331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870856331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870879922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5412055253982544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870879922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870879923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870888131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1400018185377121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870904485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351039707660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870921008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431549072265625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870937988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505688309669495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870937989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870937989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870961535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.546598494052887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870961536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870961536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870977780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349261850118637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870994206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294580578804016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871010657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205139338970184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871019441, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871019442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871043518, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5526149868965149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871043518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871043518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871051670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230308890342712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871067748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051572442054749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871083568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562801480293274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871099713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197320699691772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871099713, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871099714, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871123506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5593355894088745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871123507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871123507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871139672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411077857017517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871156373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963411211967468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871172729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13137181103229523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871180746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871180746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871204365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5676172971725464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871204366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871204366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871212228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162961602210999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871228246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299332082271576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871244433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236156105995178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871260917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131648987531662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871260918, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871260918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871284503, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5768386721611023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871284504, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871284504, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871300574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310787796974182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871316585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12695489823818207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871332533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296812772750854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871340699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871340699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871363858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5856000185012817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871363858, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871363859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871372030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13349094986915588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871387957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568312883377075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871404416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546953558921814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871420743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133439421653748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871420743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871420744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871443761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5953964591026306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871443761, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871443762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871459626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12787990272045135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871475625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269734799861908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871491941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261996477842331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871499840, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871499840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871523480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6013913154602051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871523481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871523481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871531549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856020033359528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871548212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254032403230667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871564307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12821558117866516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871580359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12643548846244812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871580360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871580360, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871603727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6106206774711609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871603728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871603728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871619541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12463903427124023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871635792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12786027789115906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871651831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13057392835617065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871659855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871659856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871683541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6196419596672058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871683542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871683542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871691906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884750962257385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871708581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12399516999721527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871725186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891492247581482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871741408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12350916862487793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871741409, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871741409, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871765023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6298641562461853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871765023, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871765023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871781619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264992356300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871797918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12775784730911255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871814763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12541553378105164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871823208, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871823208, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871846142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6428177356719971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871846142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871846143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871854334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12492509186267853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871870490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12481343001127243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871886915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12466108053922653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871903342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12412206083536148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871903343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871903343, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871926951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6552293300628662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871926951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871926951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871943514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11726808547973633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871959827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778176575899124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871975920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563322693109512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871983979, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784871983979, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872006942, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6620043516159058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872006943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872006943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872014852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11753008514642715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872031144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455973982810974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872047355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194625273346901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872063390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648675054311752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872063391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872063391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872087354, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6889047622680664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872087355, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872087355, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872104339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11845828592777252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872120362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932497680187225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872136439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11548402905464172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872144347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872144348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872168084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7180413007736206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872168084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872168084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872175920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636011302471161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872192279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633143573999405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872208645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107916384935379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872224588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446774750947952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872224661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872224662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872249428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7412678003311157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872249428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872249428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872265646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314014345407486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872281674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301463842391968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872298004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603693127632141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872306111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872306112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872329576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.75250244140625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872329577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872329577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872337686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042304337024689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872354179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135181784629822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872370459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833243280649185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872387034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114495247602463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872387035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872387035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872410532, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7620359063148499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872410533, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872410533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872426855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977653414011002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872443131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904458165168762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872459549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302250623703003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872467679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872467679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872491517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7648057341575623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872491517, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872491518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872499525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226615518331528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872516099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115804314613342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872532469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516249388456345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872548860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855589807033539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872548861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872548861, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872573547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7666770219802856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872573548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872573548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872590089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235357820987701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872606426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899011790752411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872622788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727697610855103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872630834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872630834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872654333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.767882227897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872654334, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872654334, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872662502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815460979938507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872679232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019998043775558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872696030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105614900588989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872712062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402505844831467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872712062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872712063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872736843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768605649471283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872736844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872736844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872754626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389748752117157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872771356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11809904128313065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872788155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488141119480133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872796441, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872796441, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872819227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680383920669556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872819228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872819228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872827390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397811025381088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872844193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413004249334335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872860778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151266098022461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872877133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308296769857407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872877133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872877134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872901798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691541314125061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872901799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872901799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872918524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173432469367981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872934973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746994405984879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872951697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999585688114166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872959980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872959980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872983264, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694912552833557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872983265, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872983265, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784872991605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11441735923290253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873008365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514928191900253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873025067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107366532087326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873042173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387398302555084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873042173, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873042174, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873065615, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705880999565125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873065616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873065616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873081712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610837489366531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873098240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539531707763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873114503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575135797262192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873122715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873122716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873146943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770862340927124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873146944, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873146944, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873155142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354553908109665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873171883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105423420667648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873188306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071772500872612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873205215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242979764938354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873205216, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873205216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873228963, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712953090667725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873228964, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873228964, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873245622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641881823539734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873261931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611210018396378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873278454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143525898456573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873286587, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873286587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873310092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716472148895264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873310092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873310092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873318380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253874957561493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873335010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099146515130997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873351847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075311154127121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873368151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389666259288788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873368151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873368151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873393083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724156379699707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873393083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873393083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873409894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059865653514862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873426384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610627382993698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873442325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064484566450119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873450645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873450646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873474378, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729170918464661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873474378, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873474378, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873482513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674449801445007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873498650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943196177482605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873515194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758432745933533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873531746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412043333053589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873531746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873531746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873555741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730358242988586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873555742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873555742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873571806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764424502849579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873589020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763980448246002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873605708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423363000154495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873613908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873613908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873637150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730283737182617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873637151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873637151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873645205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904991626739502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873661968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726580023765564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873678405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145062744617462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873695179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778208822011948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873695180, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873695180, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873718818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734581828117371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873718819, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873718819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873735511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501052439212799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873751721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177071392536163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873768449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765264928340912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873776882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873776882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873800969, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738660573959351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873800969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873800969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873809036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378532856702805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873825067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889679402112961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873841478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141795739531517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873857748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037707328796387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873857749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873857749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873881930, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738699913024902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873881931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873881931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873898210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063421368598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873914914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840026289224625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873931328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784465074539185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873939361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873939362, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873963040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742166519165039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873963041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873963041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784873971827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958980768918991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873988783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416039615869522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874005732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206736415624619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874022611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256551742553711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874022611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874022611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874046561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739840745925903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874046562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874046562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874063300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971637815237045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874080527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351051926612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874097572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101815328001976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874105934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874105934, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874129300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774037778377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874129300, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874129300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874137632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057361215353012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874155224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910207033157349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874171937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251810938119888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874188641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066124588251114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874188642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874188642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874212054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774360179901123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874212055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874212055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874228805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099185049533844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874245889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251439154148102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874262877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659518629312515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874271370, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874271370, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874296147, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747328877449036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874296148, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874296148, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874304287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445218324661255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874321635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752061009407043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874338131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109102725982666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874354750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675425082445145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874354751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874354752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874378724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754207849502563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874378725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874378725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874395361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469962656497955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874411940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061108261346817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874428920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504664480686188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874437514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874437514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874461764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756708264350891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874461765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874461765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874470163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108424112200737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874486548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305130481719971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874503919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496752709150314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874520580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716409981250763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874520580, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874520581, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874543815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757408022880554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874543816, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874543816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874560058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002260074019432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874576731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355134308338165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874593240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299888253211975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874601475, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874601476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874624624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759867310523987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874624624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874624625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874632991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373096168041229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874650163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248906165361404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874666660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135580986738205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874683431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268759727478027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874683431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874683432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874707218, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759820818901062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874707219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874707219, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874724120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676568746566772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874740770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714593529701233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874757550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144971311092377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874765937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874765938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed147670635.log b/recommendation_v4/rcp_logs/gbs_32768/seed147670635.log new file mode 100644 index 000000000..817b65e98 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed147670635.log @@ -0,0 +1,662 @@ +:::MLLOG {"namespace": "", "time_ms": 1784893931704, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784893931705, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784893959819, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "seed", "value": 147670635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784893959821, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784893959822, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784893959822, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784893959849, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784893959849, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784894870277, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784894870277, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784894870625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784895097409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852256536483765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784895113030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138487309217453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784895128224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384914517402649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784895135869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895135869, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895213256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500963032245636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895213257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895213257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784895220719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843511044979095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895235564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835781812667847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895250883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852310180664062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895266115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836170732975006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895266115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895266115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895289413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010941624641418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895289414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895289415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784895304588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138079434633255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895319802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823965191841125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895335198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800527155399323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895342602, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895342603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895366436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012567043304443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895366437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895366437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784895374300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382782906293869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895389459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821950554847717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895404976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823376595973969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895420322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774409890174866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895420322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895420322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895443798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014377236366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895443799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895443799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784895458897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820523023605347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895473941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747933506965637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895489241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783982396125793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895496842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895496842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895520432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016515851020813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895520433, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895520434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784895527799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700108230113983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895543178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722437620162964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895557962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367916762828827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895573290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701564073562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895573291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895573291, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895597757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018600821495056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895597758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895597758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784895612760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369762420654297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895628023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371479481458664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895643728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362568438053131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895651251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895651251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895675019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020810961723328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895675019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895675020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784895682656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702823221683502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895698870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361314207315445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895714359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364269256591797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895730536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696961104869843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895730537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895730537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895754122, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023696422576904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895754123, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895754123, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784895769305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360345184803009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895785307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13483795523643494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784895800434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651731610298157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895808027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895808027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895832030, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026381015777588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895832031, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895832031, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784895839645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293305039405823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895855386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446690142154694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895870444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486433029174805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895885558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504204154014587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895885630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895885630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895909974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029171109199524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895909974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895909974, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784895925167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336742788553238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895940288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440456986427307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895955457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510270416736603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784895963838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895963838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895987427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032342076301575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895987427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895987428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784895995041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13578671216964722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896010418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347689926624298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896025880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318744719028473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896041792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456213474273682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896041792, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896041792, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896064435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036888718605042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896064435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896064436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784896079910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336183398962021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896095598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526363670825958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896111982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505816459655762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896119890, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896119890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896143110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045623183250427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896143111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896143111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784896150805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365327090024948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896166478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345322012901306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896182654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343473643064499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896198070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320655196905136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896198140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896198140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896220820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055482387542725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896220821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896220821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784896236165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279137015342712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896252185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119813799858093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896267534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309703767299652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896275266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896275267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896297760, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5065887570381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896297761, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896297761, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784896305417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843144476413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896320929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103938102722168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896336910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298784762620926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896352317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039827346801758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896352318, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896352318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896375469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507820188999176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896375470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896375470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784896390835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285544514656067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896406357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297694444656372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896422655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13066092133522034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896430330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896430330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896453035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5091142058372498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896453036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896453036, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784896460753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150924444198608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896476036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350899338722229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896491686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321739435195923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896507833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557414710521698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896507834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896507834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896530922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5105797648429871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896530923, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896530923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784896546365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350288689136505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896561859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259193301200867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896577384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315552473068237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896585071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896585071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896608484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.512100338935852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896608485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896608485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784896617223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639819622039795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896633164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524311780929565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896648987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363169252872467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896665050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364056020975113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896665051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896665051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896688923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5136758089065552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896688923, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896688924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784896705530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346978098154068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896721376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093256950378418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896737056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362392008304596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896745083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896745083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896768831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5152435898780823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896768832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896768832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784896776650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604652881622314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896793099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255569338798523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896808822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264665007591248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896824336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075962662696838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896824337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896824337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896847886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5167389512062073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896847887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896847887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784896863539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310567706823349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896879810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816577196121216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896895623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12962199747562408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896903324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896903324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896925914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5184480547904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896925915, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896925915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784896933505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293485164642334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896949359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477648794651031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896965076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345919370651245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896981425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027849793434143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784896981426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784896981426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784897004343, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5205942392349243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784897004344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784897004344, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784897019832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281328797340393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897035474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183604180812836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897051391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551270961761475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897059397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897059397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897082764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.522999107837677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897082764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897082765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784897090618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650383055210114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897106835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335427463054657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897122482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206198811531067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897138009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091826438903809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897138010, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897138010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897161771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5259330868721008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897161771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897161772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784897177751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954136729240417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897193599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239799439907074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897209970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534800708293915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897218055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897218055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897240666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5292792916297913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897240667, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897240667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784897248506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13186874985694885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897264652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340707123279572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897280622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493117690086365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897296665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345319151878357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897296665, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897296665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897320534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.533307671546936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897320535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897320535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784897336672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358770728111267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897353356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397452235221863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897369561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303044438362122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897377513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897377514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897401184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5381539463996887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897401185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897401185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784897409467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14022080600261688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897425906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352008432149887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897442540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432572782039642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897459331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135033518075943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897459331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897459332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897482791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.543677031993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897482791, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897482792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784897498877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504838943481445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897515136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330607533454895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897531396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230332732200623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897540024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897540024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897564642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5496487617492676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897564642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897564643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784897572838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323358416557312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897588978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064241409301758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897604923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356230229139328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897621173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214121758937836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897621174, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897621174, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897644786, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5568435192108154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897644787, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897644787, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784897661000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437968492507935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897677561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969988584518433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897693721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13130205869674683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897701651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897701651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897725888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.565775990486145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897725888, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897725889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784897733610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183937966823578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897749634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006800413131714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897765718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324712038040161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897782213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185124099254608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897782214, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897782214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897806097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5753545761108398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897806098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897806098, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784897822248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312178075313568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897838176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272422820329666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897853812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330271065235138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897861857, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897861857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897885077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5846254825592041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897885078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897885078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784897893171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346035778522491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897909117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12567263841629028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897925519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546154856681824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897941928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13131502270698547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897941929, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897941929, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897965530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5975090265274048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897965531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897965531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784897981461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783822417259216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784897997402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697544693946838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898013733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649433314800262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898021631, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898021631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898045443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6037235260009766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898045444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898045444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784898053531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856747210025787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898070120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1251889020204544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898086091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280025690793991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898102198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12641173601150513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898102199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898102199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898125364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6158297657966614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898125365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898125365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784898141210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12431211769580841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898157503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747898697853088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898173482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015219569206238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898181448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898181448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898205802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6292466521263123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898205803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898205803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784898214373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847375869750977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898231136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12376555055379868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898247874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12864044308662415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898264205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311524152755737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898264206, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898264206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898288112, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.645656943321228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898288113, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898288113, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784898304884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257796436548233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898321332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12709352374076843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898338034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12527740001678467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898346368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898346368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898369661, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6691115498542786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898369662, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898369662, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784898377740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418829649686813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898393718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12438581883907318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898409995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402839213609695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898426256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123397096991539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898426257, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898426257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898450176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6920463442802429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898450177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898450177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784898466913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11687647551298141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898483085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11629381775856018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898499006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469613760709763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898507024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898507024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898530086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7032926082611084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898530087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898530087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784898537905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603941023349762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898554177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318821460008621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898570397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847894638776779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898586212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541688442230225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898586212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898586212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898610455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7327128052711487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898610455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898610455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784898627225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11702260375022888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898643017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767310857772827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898658851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355894804000854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898666674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898666674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898690040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7458020448684692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898690041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898690041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784898697832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491900682449341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898714045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420338600873947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898730201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962509363889694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898745949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239200085401535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898746024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898746024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898770926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7510687112808228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898770927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898770927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784898787208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074003577232361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898803251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140166223049164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898819534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043093129992485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898827552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898827552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898851451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7528185844421387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898851452, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898851452, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784898859490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087883710861206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898875898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846098512411118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898892243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580871254205704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898908941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888613015413284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898908942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898908942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898932511, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7587505578994751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898932512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898932512, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784898948601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718685388565063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898964533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658130794763565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898980784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190325021743774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784898988795, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784898988795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784899012234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.759536862373352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784899012235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784899012235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784899020130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168871283531189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899036619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028319597244263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899052903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044854149222374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899069399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730867832899094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899069400, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899069400, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899093280, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7612096667289734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899093281, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899093281, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784899109559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149907857179642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899125722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883177071809769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899141741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736341774463654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899149586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899149586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899172908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.764346182346344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899172909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899172909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784899181058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814838856458664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899197332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950808972120285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899213831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098454892635345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899229688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457506030797958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899229688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899229688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899253767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640153169631958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899253768, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899253768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784899271083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437021195888519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899287683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11826812475919724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899304550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1149819865822792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899312916, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899312916, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899336179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7681159377098083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899336180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899336180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784899344407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470068991184235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899361246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415497958660126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899377831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187861114740372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899394245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11321951448917389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899394246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899394246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899418992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679373025894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899418993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899418993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784899435555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698412895202637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899451948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169692650437355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899468723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011853814125061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899477079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899477079, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899500883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.767989993095398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899500884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899500884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784899509257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503450572490692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899526127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480952054262161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899542816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055590212345123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899560056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040646880865097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899560057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899560057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899583799, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679269313812256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899583799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899583799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784899599752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659617185592651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899616187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054217740893364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899632403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546448081731796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899640520, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899640520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899663990, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680603861808777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899663990, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899663991, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784899672149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395914316177368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899688770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140336841344833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899705272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736186057329178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899721995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233330518007278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899721995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899721996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899746501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688477635383606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899746502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899746503, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784899763219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669861733913422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899779480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640697181224823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899795893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160395503044128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899803882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899803883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899827938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695965766906738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899827938, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899827939, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784899836198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273171961307526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899852745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119348764419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899869454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089996248483658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899885502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378745943307877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899885503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899885503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899910022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692160606384277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899910022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899910022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784899926608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572392493486404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899943187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630445182323456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899959314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625782608985901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784899967742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899967742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899991120, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696937918663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899991121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899991121, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784899999330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723436623811722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900015503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996604636311531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900032064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763821750879288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900048748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450759530067444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900048748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900048748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900072360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706308960914612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900072361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900072361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784900088574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796477645635605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900105717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785935819149017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900122397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411620140075684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900130639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900130639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900154171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719217538833618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900154172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900154172, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784900162291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091800406575203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900178946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715372860431671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900195274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172036081552505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900211862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789933055639267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900211862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900211862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900235209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720355987548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900235210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900235210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784900251802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10524765402078629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900267940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210694372653961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900284936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775960981845856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900293371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900293371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900316753, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716939449310303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900316754, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900316754, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784900324845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044904962182045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900340856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926146268844604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900357318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450066417455673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900373652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102755069732666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900373653, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900373653, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900397188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724789381027222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900397188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900397189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784900413582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617899894714355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900430405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843904316425323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900447007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080745980143547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900455122, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900455123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900479271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725208401679993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900479272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900479272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784900488073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987064242362976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900504837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465753614902496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900521607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199462413787842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900538393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328709125518799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900538394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900538394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900562121, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735397219657898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900562121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900562122, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784900578782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961014777421951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900595817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376698315143585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900612691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032286286354065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900621084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900621084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900644551, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774141788482666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900644553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900644553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784900652834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041593551635742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900670340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10884043574333191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900687105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251145601272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900703942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085303127765656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900703943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900703943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900726948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736579179763794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900726949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900726949, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784900743699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013428121805191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900760903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248345673084259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900777910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676716804504395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900786419, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900786419, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900809727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743808627128601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900809727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900809727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784900817892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452975869178772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900835410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751271992921829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900851962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104171186685562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900868634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696075856685638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900868634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900868634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900892216, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723627090454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900892217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900892217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784900908856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446551442146301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900925411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647419095039368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900942320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540623962879181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900950874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900950874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900973844, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739356756210327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900973845, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900973845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784900982201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848148912191391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784900998504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315167903900146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901015939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477735102176666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901032858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736958682537079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901032859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901032859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901056310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741047143936157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901056311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901056311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784901072635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001630499958992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901089331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037607491016388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901105990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295327007770538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901114275, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901114275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901138458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743462920188904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901138459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901138459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784901146818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380604863166809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901164067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249249637126923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901180653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331678181886673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901197802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316306352615356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784901197803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784901197803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed162052595.log b/recommendation_v4/rcp_logs/gbs_32768/seed162052595.log new file mode 100644 index 000000000..f18e10344 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed162052595.log @@ -0,0 +1,685 @@ +:::MLLOG {"namespace": "", "time_ms": 1784874656304, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784874656304, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784874679946, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "seed", "value": 162052595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784874679950, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784874679951, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784874680074, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784874680074, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784875572346, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784875572347, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784875572674, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784875720888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139136403799057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784875736276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897855579853058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784875751281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390974223613739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784875758721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875758721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875809938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49946528673171997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875809939, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875809939, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784875817312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905450701713562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875831991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390204131603241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875847093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890133798122406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875862159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885723054409027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875862159, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875862160, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875885147, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4996291995048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875885148, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875885148, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784875900270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884124159812927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875915334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883893191814423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875930494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872233033180237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875937806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875937806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875961435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49983853101730347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875961436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875961436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784875969438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387232393026352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875984470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860827684402466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784875999844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857673108577728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876015231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839475810527802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876015231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876015232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876038219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000891089439392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876038220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876038220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784876053071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858966529369354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876068144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812458515167236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876083499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822151720523834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876090900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876090901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876114054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004041790962219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876114055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876114055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784876121285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766759634017944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876136646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377570778131485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876151501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745611906051636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876166686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747118413448334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876166687, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876166687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876190722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007785558700562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876190723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876190723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784876205362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760077953338623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876220355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747134804725647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876236090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13680291175842285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876243598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876243598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876266884, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5011768937110901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876266885, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876266885, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784876274471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742958009243011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876290707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662174344062805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876305869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683035969734192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876321944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733302056789398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876321945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876321945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876344981, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016996264457703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876344982, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876344982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784876360116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365136206150055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876375791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519632816314697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784876390806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368206888437271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876398354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876398355, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876422816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022421479225159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876422817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876422817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784876430410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336698830127716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876445988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501881062984467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876460839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522186875343323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876475794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354401707649231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876475869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876475870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876500058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028268694877625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876500059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876500059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784876515190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339917927980423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876530176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485319912433624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876545191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353011429309845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876553620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876553621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876577295, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034870505332947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876577296, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876577296, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784876584586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357787400484085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876599500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498300313949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876614688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219957053661346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876630782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346915066242218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876630783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876630783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876653656, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042855739593506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876653656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876653657, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784876668984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384021818637848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876684608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534656167030334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876700569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498808443546295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876708297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876708298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876731966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052836537361145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876731969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876731969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784876739556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649600744247437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876754985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346731662750244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876771137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343257874250412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876786521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202841579914093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876786590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876786591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876809624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062920451164246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876809624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876809624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784876824839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328946202993393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876840837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312018781900406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876855954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299345970153809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876863545, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876863546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876886681, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074412226676941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876886681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876886681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784876894232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842402398586273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876909541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110655546188354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876925391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298253834247589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876940778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13048326969146729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876940779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876940779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876963995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086959600448608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876963996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876963996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784876979405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275139033794403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876995004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969368696212769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877011265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065853714942932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877018773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877018774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877041843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100130438804626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877041843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877041843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784877049527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315067857503891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877064700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517636060714722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877080258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323339819908142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877096207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356988400220871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877096207, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877096208, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877119834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114462375640869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877119835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877119835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784877135360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526105880737305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877150871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258680701255798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877166470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323065638542175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877174171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877174172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877198155, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5127866864204407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877198156, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877198156, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784877206943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637754321098328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877222909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528980314731598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877238869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362985074520111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877254998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647182285785675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877254998, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877254999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877278968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5140691995620728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877278968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877278969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784877295652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482147455215454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877311734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085411489009857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877327590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361377090215683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877335591, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877335591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877359852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5154426097869873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877359853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877359853, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784877367656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359340250492096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877384221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323823481798172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877400185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326555758714676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877415888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309511810541153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877415888, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877415889, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877439295, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5169503092765808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877439295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877439295, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784877454928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311296820640564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877471142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382627934217453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877486767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952865660190582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877494537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877494537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877517649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5186318159103394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877517650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877517651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784877525353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943795323371887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877541149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479670882225037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877556778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455377519130707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877573467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044559955596924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877573468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877573468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877596721, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5204984545707703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877596722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877596722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784877612287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281389445066452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877627873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183456659317017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877643467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353229284286499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877651407, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877651408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877675034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5225229859352112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877675035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877675035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784877682863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366700828075409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877699007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358429074287415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877714448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321735978126526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877729798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096463680267334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877729798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877729799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877752916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.524826169013977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877752916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877752917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784877768459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961654365062714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877784038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243579864501953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877800383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535983860492706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877808464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877808465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877831309, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.527394711971283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877831309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877831310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784877839074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197562098503113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877854864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348931074142456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877870688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505880534648895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877886562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465414941310883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877886563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877886563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877910913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5303020477294922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877910914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877910914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784877926883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360570788383484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877943353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398416340351105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877959399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312938809394836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877967332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877967332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877990950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5335052609443665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877990951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877990951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784877999144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403648853302002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878015564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529512286186218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878032179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443440198898315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878048847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352185606956482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878048847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878048848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878071673, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5369900465011597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878071673, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878071674, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784878087481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519394397735596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878103446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324449956417084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878119373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242632150650024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878127934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878127934, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878151918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.540977954864502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878151919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878151919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784878159947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253335654735565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878175816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083170354366302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878191559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588334619998932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878207686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234944641590118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878207686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878207686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878231012, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5458800792694092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878231013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878231013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784878247115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481681048870087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878263550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031920790672302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878279310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193704187870026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878287101, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878287101, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878309786, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5522220730781555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878309787, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878309787, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784878317465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231487572193146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878333183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084672391414642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878349177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316617906093597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878365653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269564509391785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878365654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878365655, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878389469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.560624897480011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878389470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878389470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784878405605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214902579784393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878421717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820085883140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878437387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415174186229706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878445497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878445498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878468864, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5701613426208496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878468865, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878468865, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784878476985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467752933502197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878492920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267929971218109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878509301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692049145698547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878525232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305294513702393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878525233, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878525233, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878548148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.580833911895752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878548149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878548149, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784878563915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945358455181122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878579813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861892580986023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878595980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12803718447685242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878603763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878603764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878626744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5907667279243469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878626745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878626745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784878634820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301960051059723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878651204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127642422914505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878667173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019591569900513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878683058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12845537066459656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878683059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878683059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878706330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6009939908981323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878706330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878706330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784878722070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270119547843933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878738378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12985949218273163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878754337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321232795715332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878762292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878762292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878786123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6097543835639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878786123, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878786123, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784878794679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131648987531662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878811294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670856714248657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878827559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133926689624786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878843729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666799128055573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878843729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878843729, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878867130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6181161999702454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878867130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878867130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784878883757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12895919382572174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878900063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301700472831726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878916975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281658560037613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878925188, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878925189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878947975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6264981627464294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878947975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878947975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784878955995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763278186321259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878972085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747254967689514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878988364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809044122695923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879004645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12735523283481598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879004646, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879004646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879027750, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6338135600090027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879027751, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879027751, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784879044244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12173520773649216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879060483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12080155313014984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879076377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192319318652153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879084373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879084374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879106877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6373912692070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879106878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879106878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784879114685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12123046070337296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879130783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787310242652893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879146905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12247320264577866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879162788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956541240215302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879162788, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879162789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879186585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6524298191070557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879186586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879186586, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784879203218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12100406736135483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879218994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224818974733353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879234915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11818795651197433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879242809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879242810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879266619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6693538427352905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879266619, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879266620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784879274364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878838390111923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879290478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11891284584999084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879306660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355441063642502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879322449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727797985076904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879322523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879322524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879347564, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6964111924171448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879347564, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879347565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784879363720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616040766239166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879379716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631593853235245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879395955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925589501857758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879403971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879403972, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879428265, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7233104705810547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879428265, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879428266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784879436353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11295989900827408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879452914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456789821386337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879469200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123288944363594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879485569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466862261295319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879485570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879485570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879509469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7477177381515503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879509469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879509470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784879525470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341744661331177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879541375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278784275054932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879557434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11687988042831421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879565364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879565365, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879589118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7550884485244751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879589119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879589119, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784879596975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695717483758926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879613075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430208384990692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879628944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080881655216217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879645072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270922422409058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879645072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879645073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879669830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7600334286689758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879669830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879669830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784879686320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509367823600769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879702564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130475252866745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879718679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111073337495327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879726702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879726702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879750893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628577351570129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879750894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879750894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784879759003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134184151887894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879775446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283501982688904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879792049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287983506917953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879807981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751734673976898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879807982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879807982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879831595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7646251916885376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879831596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879831596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784879848757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568867415189743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879864859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11994699388742447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879881112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763225495815277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879889217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879889218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879912406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7653693556785583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879912406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879912407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784879920348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592942476272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879936817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11562953889369965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879953024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292555183172226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879969105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377976834774017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879969105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784879969106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784879993430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7660014033317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784879993431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784879993431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784880010168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11853061616420746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880026710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773885786533356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880043471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099152266979218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880051855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880051855, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880075221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7665657997131348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880075222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880075222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784880083665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578910052776337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880100499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155736893415451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880117065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120844632387161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880134096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460948199033737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880134096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880134097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880156875, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7672369480133057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880156876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880156876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784880172765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672132670879364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880189037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604295134544373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880205292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621634125709534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880213448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880213449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880236991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677485346794128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880236991, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880236992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784880245205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419361293315887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880261949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111544169485569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880278377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761044919490814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880295395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285081505775452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880295395, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880295396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880318947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685725092887878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880318948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880318948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784880335527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694804042577744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880351675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631028562784195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880367962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021115630865097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880375953, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880375953, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880400201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692181468009949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880400202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880400202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784880408383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024765595793724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880424860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133492946624756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880441389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11165677756071091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880457409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039206013083458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880457410, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880457410, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880481946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699382305145264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880481946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880481946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784880498437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663795471191406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880514913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712285339832306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880530866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720464587211609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880539193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880539194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880563178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704401016235352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880563179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880563179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784880571396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768880695104599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880587560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955427050590515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880604268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078442633152008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880620930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473290085792542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880620930, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880620931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880645630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710413932800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880645631, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880645631, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784880661816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076197698712349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880678849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801464319229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880695398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044226884841919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880703585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880703586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880727020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713555097579956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880727021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880727021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784880734963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871867090463638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880751378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076890230178833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880767648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172975599765778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880784166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804307460784912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880784167, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880784167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880807745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717275023460388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880807746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880807746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784880824296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519541800022125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880840359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188403725624084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880857352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813146084547043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880865852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880865852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880889768, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721058130264282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880889768, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880889768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784880897880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448190569877625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880914130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934572875499725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880930690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410385370254517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880947184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135473102331161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880947185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880947185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880971622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724032998085022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880971623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880971623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784880988239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647058486938477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881005199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862186551094055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881021796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850712656974792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881029950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881029951, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881054457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726014256477356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881054457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881054457, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784881063272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101488247513771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881080220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467408388853073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881097049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123494952917099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881113936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316981166601181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881113936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881113937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881138141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725335359573364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881138141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881138141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784881154878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001693457365036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881172037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351300776004791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881188979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064265668392181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881197388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881197388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881221612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728598117828369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881221612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881221612, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784881229818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063958704471588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881247154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942143946886063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881263803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240100860595703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881280467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124078184366226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881280467, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881280468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881303813, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732689380645752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881303813, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881303814, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784881320488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008694022893906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881337530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285825818777084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881354308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707936972379684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881362726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881362726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881387576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732497453689575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881387577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881387577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784881395674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484773457050323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881412864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761266946792603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881429287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10130003839731216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881445648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698056221008301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881445648, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881445649, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881469293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773747980594635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881469294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881469294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784881485725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485746711492538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881502024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611388087272644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881518764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533851385116577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881527253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881527254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881550623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738913893699646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881550623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881550624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784881558966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086914986371994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881575225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305783897638321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881592473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475026071071625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881609206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715780407190323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881609207, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881609207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881632656, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739665508270264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881632657, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881632657, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784881648654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071577876806259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881665058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10376092791557312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881681376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344579070806503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881689453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881689453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881712649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774172306060791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881712650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881712650, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784881720868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401216894388199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881737834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029793992638588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881754344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406856775283813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881771257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318776965141296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881771258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881771258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881794883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742573022842407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881794883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881794883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784881811548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716130584478378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881828084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075611412525177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881844541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174486041069031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881852723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881852723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881876374, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743743658065796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881876375, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881876375, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784881884557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137377679347992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881901123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038921236991882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881917934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604315251111984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881934808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288899391889572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881934808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881934809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881958614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745136618614197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881958615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881958615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784881974822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770188271999359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784881992267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035032868385315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784882008879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598792135715485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed182111600.log b/recommendation_v4/rcp_logs/gbs_32768/seed182111600.log new file mode 100644 index 000000000..ff3b503ef --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed182111600.log @@ -0,0 +1,1410 @@ +:::MLLOG {"namespace": "", "time_ms": 1784846531479, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784846531480, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784846554730, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "seed", "value": 182111600, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784846554732, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784846555027, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784846555027, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784847421337, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784847421339, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784847421628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784847591917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874059915542603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847607593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387520283460617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847622192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866497576236725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784847629473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847629474, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847653204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4952045679092407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847653205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847653205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784847660504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862104713916779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847675213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879913091659546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847690417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386469453573227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847705583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865526020526886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847705584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847705584, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847728541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4953469932079315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847728542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847728542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784847743422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858963549137115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847758215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852186501026154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847773338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836008310317993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847780698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847780699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847804452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4955340027809143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847804453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847804453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784847812141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852718472480774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847827195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384502798318863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847842610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843536376953125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847857762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814565539360046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847857763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847857763, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847880778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4957517385482788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847880779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847880779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784847895692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138319194316864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847910566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792499899864197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847925630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812458515167236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847933057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847933058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847956373, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49601995944976807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847956374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847956374, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784847963598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753008842468262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847978806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744856417179108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784847993488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723765313625336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848008591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371459662914276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848008592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848008593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848032278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.496308296918869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848032280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848032280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784848046896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374565213918686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848061816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732826709747314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848077649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367386132478714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848085146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848085146, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848108811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49664002656936646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848108811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848108811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784848116342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729068636894226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848132498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650794327259064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848147587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136808842420578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848163583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13709215819835663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848163584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848163584, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848186531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49704262614250183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848186532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848186532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784848201670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364547312259674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848217395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543549180030823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784848232363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678905367851257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848239941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848239942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848262910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.497429221868515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848262912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848262912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784848270455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133612260222435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848285921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350184828042984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848300815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525837659835815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848315720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536915183067322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848315798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848315799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848339998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49785351753234863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848339998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848339999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784848355081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13414955139160156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848370174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487350940704346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848385169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536527752876282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848393401, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848393402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848416738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4983193576335907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848416738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848416738, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784848424065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358182430267334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848438988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508230447769165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848454158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324046105146408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848470042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346844583749771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848470043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848470043, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848492520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989452660083771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848492520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848492520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784848507917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397303223609924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848523517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531820476055145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848539413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513554632663727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848547298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848547299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848570683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49988052248954773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848570683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848570683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784848578326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641014695167542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848593913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335371434688568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848610347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343303769826889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848625838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219481706619263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848625910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848625911, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848649701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009028911590576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848649702, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848649702, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784848665037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297244906425476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848681031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313360035419464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848696228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331281065940857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848703887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848703888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848727245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020161867141724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848727246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848727246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784848734849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383158266544342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848750228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131073459982872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848766123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997221946716309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848781610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304538995027542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848781610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848781611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848805041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032609701156616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848805042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848805042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784848820519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279670476913452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848836198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298374980688095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848852574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13066096603870392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848860170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848860171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848883380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046176910400391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848883381, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848883381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784848891016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170096278190613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848906222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515371084213257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848921879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314726948738098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848937756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561691343784332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848937757, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848937757, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848960839, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.506084144115448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848960839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848960840, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784848976356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522502779960632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784848992041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260427117347717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849007700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327214121818542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849015455, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849015456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849039246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5075395107269287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849039247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849039247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784849047701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643869757652283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849063630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353423297405243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849079592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622204959392548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849095685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364784687757492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849095685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849095686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849119767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5089837312698364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849119767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849119768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784849136420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348285973072052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849152667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130825936794281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849168557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626834750175476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849176661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849176661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849200570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5104460120201111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849200571, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849200571, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784849208417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593967258930206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849224473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325288563966751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849239883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13267068564891815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849255247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093504309654236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849255247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849255248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849278982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5120187401771545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849278983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849278983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784849294540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311337649822235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849310712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382880061864853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849326305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963393330574036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849334069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849334070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849357521, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138807892799377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849357522, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849357522, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784849365153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941116094589233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849380810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476358354091644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849396549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345067173242569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849412834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038606941699982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849412835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849412835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849436367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5162106156349182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849436367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849436368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784849451647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12804071605205536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849467100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196001946926117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849482717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516877591609955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849490651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849490652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849514788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5187477469444275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849514788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849514789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784849522595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659456372261047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849538880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355419039726257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849554669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209617137908936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849570332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096198439598083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849570332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849570333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849593759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5216954350471497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849593759, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849593759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784849609626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960633635520935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849625489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253825902938843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849641695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533185422420502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849649694, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849649695, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849673066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5251766443252563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849673066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849673066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784849680848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197404146194458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849696787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337263464927673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849712644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494770228862762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849728522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458283245563507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849728522, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849728523, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849752576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.529434084892273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849752577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849752577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784849768576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135977104306221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849785278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339210718870163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849801631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331060230731964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849809630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849809631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849832841, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5341885685920715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849832842, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849832842, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784849841076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14020708203315735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849857481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352551430463791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849874106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432583212852478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849890694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351906955242157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849890694, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849890695, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849913584, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5395292043685913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849913584, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849913584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784849929704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351621448993683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849945871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332181692123413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849962022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221514225006104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784849970674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784849970675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784849994377, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5453697443008423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784849994378, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784849994378, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784850002535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132341206073761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850018461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072775304317474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850034179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13570399582386017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850050247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214674592018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850050248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850050248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850074100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5523521900177002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850074100, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850074101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784850090089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437247276306152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850106729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997815012931824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850122641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165324926376343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850130523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850130524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850154743, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5610808730125427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850154744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850154744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784850162572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202570378780365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850178621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025768101215363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850194799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326046586036682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850211245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212110102176666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850211246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850211246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850235531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5715835690498352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850235532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850235532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784850251590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316252499818802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850267549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274266093969345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850283126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333703249692917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850291090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850291091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850314819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.581810712814331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850314820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850314820, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784850322901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339533030986786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850338806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606078386306763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850354844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12592929601669312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850370987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199320435523987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850370988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850370988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850395004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.595130980014801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850395005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850395006, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784850410862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12836328148841858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850426852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127407506108284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850443075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12682726979255676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850450919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850450919, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850475024, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6025228500366211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850475025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850475025, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784850483093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889370322227478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850499605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1260717660188675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850515477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880700826644897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850531292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270032525062561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850531292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850531293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850554652, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6162679195404053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850554653, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850554653, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784850570492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12523774802684784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850586781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281438171863556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850602751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069632649421692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850610720, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850610721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850634632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.630220890045166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850634633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850634633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784850643103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928304076194763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850659757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12454427778720856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850676102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12923333048820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850692111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12412504106760025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850692112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850692112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850715284, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6470739245414734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850715285, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850715285, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784850731757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268034279346466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850747912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774905562400818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850764324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257985383272171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850772581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850772582, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850796785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6673780679702759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850796786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850796786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784850805098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12494467198848724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850821191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12476576864719391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850837549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12483946233987808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850853829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12397348880767822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850853830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850853830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850877710, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6874874234199524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850877710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850877710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784850894252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780256032943726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850910425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792366206645966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850926175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568206548690796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850934158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850934159, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850957331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6948780417442322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850957331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850957332, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784850965152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174389198422432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850981144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432179808616638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784850997188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11931685358285904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851012972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631713062524796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851012972, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851012973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851036244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7241867184638977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851036245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851036245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784851053010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1180490031838417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851069179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900969803333282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851085370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498980224132538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851093412, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851093412, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851116619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7437043190002441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851116620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851116620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784851124482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11641532927751541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851140912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560536921024323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851157267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114698201417923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851173169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138516217470169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851173246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851173246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851197926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7570252418518066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851197927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851197927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784851214116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208604276180267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851229980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223212629556656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851246102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570789873600006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851254082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851254083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851277775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7622617483139038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851277776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851277776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784851285771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981258749961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851302068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102246642112732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851318182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704228281974792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851334511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013675481081009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851334512, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851334512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851357809, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7667514681816101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851357810, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851357810, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784851373746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083611324429512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851389619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702814161777496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851405557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203691363334656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851413452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851413453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851436764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680715322494507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851436765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851436765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784851444600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155724734067917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851460770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109599769115448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851476832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473621636629105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851493286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792173445224762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851493286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851493287, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851516645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692736387252808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851516646, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851516646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784851532916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180353164672852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851549037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090700775384903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851565160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771831125020981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851573063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851573063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851595612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699450850486755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851595613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851595613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784851603783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816849023103714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851620272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099638044834137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851636831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117807775735855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851652667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407275706529617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851652668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851652668, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851675859, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705444693565369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851675859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851675859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784851692898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377395689487457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851709267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11853807419538498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851725960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476903408765793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851734200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851734201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851757063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713190913200378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851757063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851757063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784851765227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417792737483978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851782048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356823891401291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851798418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114412397146225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851814660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281315982341766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851814661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851814661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851838833, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715410590171814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851838834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851838834, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784851855523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696930974721909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851871947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747737973928452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851888732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004626750946045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851897099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851897099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851920021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718138098716736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851920021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851920022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784851928421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476525664329529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851945197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472690105438232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851961818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088638752698898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851978992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357476770877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784851978993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784851978993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784852002299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718023061752319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784852002299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784852002300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784852018306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583936423063278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852034698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557520389556885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852050930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528592765331268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852059065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852059065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852082182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721022963523865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852082183, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852082183, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784852090323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391482710838318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852106852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099706590175629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852123127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746991634368896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852140011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174505949020386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852140011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852140012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852163419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723265886306763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852163420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852163420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784852180102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646963119506836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852196096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612708330154419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852212389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121628642082214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852220431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852220432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852243628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725151777267456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852243628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852243629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784852251798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242553055286407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852268179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010938361287117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852284735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066654324531555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852300802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377956926822662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852300802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852300803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852324861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725733518600464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852324862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852324862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784852341266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565954446792603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852357678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564178228378296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852373502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647234320640564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852381736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852381737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852405357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726770639419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852405358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852405358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784852413611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686194151639938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852429621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934306144714355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852446116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107803575694561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852462763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408821702003479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852462764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852462764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852486083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728015184402466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852486084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852486084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784852502041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734564065933228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852519023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792483389377594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852535557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393762588500977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852543695, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852543695, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852566051, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729513645172119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852566052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852566052, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784852574175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878214240074158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852590776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705874860286713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852607066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139542609453201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852623623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767091065645218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852623624, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852623624, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852646094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730157971382141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852646094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852646094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784852662851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480092465877533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852678942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175109654664993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852695696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772254317998886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852704185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852704186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852726842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730743288993835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852726843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852726843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784852735009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404272377490997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852751319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936217963695526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852767922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410480737686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852784356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048762500286102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852784356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852784357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852807881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773175835609436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852807882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852807882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784852824414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601241886615753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852841209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830448567867279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852857691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808895528316498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852865785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852865785, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852890248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732925415039062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852890249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852890249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784852899178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956227779388428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852916040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437904834747314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852932780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191356182098389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852949587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248272657394409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784852949587, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852949588, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852973042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733628749847412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852973043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852973043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784852989611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992423444986343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853006509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353226006031036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853023511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022711545228958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853031864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853031865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853055720, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734570503234863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853055721, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853055721, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784853063998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104089766740799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853081335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923752188682556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853098038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237768828868866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853114758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081299185752869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853114758, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853114759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853137531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735739350318909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853137532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853137532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784853154287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968382656574249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853171542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250720173120499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853188587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690311133861542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853197042, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853197043, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853221459, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736304402351379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853221459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853221460, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784853229589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478319764137268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853246917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771667212247849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853263543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108467191457748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853280030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669853538274765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853280030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853280031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853303821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735432386398315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853303822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853303822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784853320223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045977771282196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853336534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587982088327408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853353146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496162623167038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853361542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853361543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853385209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736853361129761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853385210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853385210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784853393489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855565965175629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853409624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329199582338333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853426759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471451282501221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853443516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709378123283386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853443517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853443518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853466664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736452221870422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853466665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853466665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784853482783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10049986839294434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853499202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384801775217056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853515487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315150022506714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853523639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853523640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853547168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737537026405334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853547169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853547169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784853555499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393078625202179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853572541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291284322738647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853589019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331601440906525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853605882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294747352600098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853605883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853605884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853629087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737916111946106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853629087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853629088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784853645623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704577714204788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853662122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729119181632996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853678673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156388580799103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853686915, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853686915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853710640, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739288806915283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853710641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853710642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784853718823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011691614985466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853735244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040774941444397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853751788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582031309604645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853768574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305297374725342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853768574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853768575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853792339, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738819122314453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853792340, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853792340, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784853808710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733947157859802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853826129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374440252780914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853842698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566699504852295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853850734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853850734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853873952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774043083190918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853873953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853873953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784853882080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307753831148148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853898602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808684676885605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853914969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023399829864502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853931619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682377219200134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853931619, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853931619, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853955396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740324139595032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853955397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853955397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784853972223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173565149307251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784853988518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645587742328644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854005288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921825468540192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854013649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854013650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854037538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740351557731628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854037539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854037539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784854046097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136828362941742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854062509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721053928136826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854078996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261761397123337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854095599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261096060276031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854095600, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854095600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854119534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740909457206726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854119535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854119535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784854135801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506537556648254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854152532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049593836069107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854169385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197260022163391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854177749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854177749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854201576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740944623947144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854201577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854201577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784854209694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693272203207016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854226567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255978792905807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854243376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907749086618423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854259949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258698999881744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854259949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854259949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854284043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742562890052795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854284043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854284044, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784854300771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887175798416138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854317486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377819627523422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854334299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11186163127422333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854342656, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854342656, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854366975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743983864784241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854366975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854366976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784854375368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227315664291382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854392125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177199333906174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854409202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730700194835663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784854426313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056552082300186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854426314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854426314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854450589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743541598320007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854450590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854450590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784854467104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397042125463486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854483629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835488140583038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854500175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113186925649643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854508609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854508610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854532855, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745516300201416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854532855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854532856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784854541340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147874593734741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854558234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403944551944733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854574767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106557622551918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854591885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867097228765488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854591886, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854591886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854615980, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745243310928345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854615981, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854615981, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784854632669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319972783327103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854649162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002250462770462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854665845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189524292945862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854674212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854674213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854697989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744430303573608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854697989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854697989, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784854706194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09970983862876892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854722727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060033202171326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854739361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245925188064575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854756206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579092800617218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854756207, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854756207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854780067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744647264480591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854780068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854780068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784854796225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043790876865387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854813352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999961942434311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854829855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613450407981873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854838034, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854838034, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854862052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744717001914978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854862053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854862053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784854870098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162927210330963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854886651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634391754865646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854902870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10059225559234619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854919151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188908874988556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854919152, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854919152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854943306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774426281452179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854943307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854943308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784854959803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198353976011276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854976684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342279076576233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784854993003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159864276647568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855001742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855001743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855025867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745714783668518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855025868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855025868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784855034210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041301041841507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855050821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069408431649208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855067244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244019329547882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855083863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10018271952867508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855083863, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855083863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855107635, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745604515075684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855107635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855107635, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784855124187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494521260261536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855140664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155381262302399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855157384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200262814760208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855165874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855165875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855189630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745813131332397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855189631, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855189631, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784855197995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206164956092834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855214356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943593084812164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855231200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765033960342407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855248006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822014510631561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855248006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855248007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855271896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746951580047607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855271897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855271897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784855288561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254458338022232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855305208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471378266811371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855321800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065443068742752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855330060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855330061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855353177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774821400642395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855353177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855353178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784855361606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505133867263794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855378562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417428612709045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855395196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728173702955246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855411968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088341549038887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855411969, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855411969, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855436081, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774836003780365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855436082, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855436082, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784855452396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844895243644714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855469576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088062971830368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855486622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075727567076683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855495007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855495007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855518276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750075459480286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855518276, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855518277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784855526541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260562390089035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855543372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032036691904068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855560296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072482168674469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855577122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657977312803268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855577123, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855577123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855600482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750773429870605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855600483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855600483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784855617509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657744109630585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855634355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506314784288406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855651555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086967587471008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855659670, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855659670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855684424, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750322818756104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855684425, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855684425, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784855692695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082935631275177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855709818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012326925992966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855726978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079300194978714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855743988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153607070446014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855743988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855743989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855767273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751931548118591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855767275, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855767275, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784855784374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457380652427673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855801483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345470696687698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855818471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130247265100479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855827085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855827086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855850299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753354907035828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855850299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855850300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784855858692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871616750955582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855875991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301910877227783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855892699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385111510753632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855910047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11856640875339508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.2131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855910048, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855910048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855933217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753516435623169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855933218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855933218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784855950493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177339032292366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855967525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356871575117111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855984617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375580728054047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784855993256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784855993257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784856016565, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755023837089539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784856016565, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784856016565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784856024953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460378766059875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856042241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515093594789505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856059497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460176110267639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856076491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183830127120018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856076491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856076492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856100877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757167220115662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856100877, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856100877, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784856118788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938702523708344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856136192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541733145713806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.2431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856153290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411824822425842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856162029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856162029, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856185351, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757127285003662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856185351, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856185352, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784856193956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063022911548615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856211023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267948150634766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856228202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792471468448639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856245508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078297346830368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856245509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856245509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856269346, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758269906044006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856269346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856269346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784856286274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377263069152832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.2631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856303590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984829813241959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856320521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842043161392212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856328957, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856328958, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856352296, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759113907814026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856352297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856352297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784856360476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623124241828918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856377566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510675609111786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856394349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458006709814072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856411256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889117419719696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.2831666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856411256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856411257, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856434060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775875985622406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856434060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856434060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784856451107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142009496688843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856468100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581836104393005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856484704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793590545654297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856493368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856493368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856517830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759811282157898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856517830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856517831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784856526129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699032992124557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856543075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067790761590004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856559652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096699982881546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.3031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856576557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826516151428223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856576557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856576558, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856599515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761326432228088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856599515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856599516, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784856616155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068604588508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856632849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528860986232758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856649956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421723872423172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856658494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856658495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856681310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760447263717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856681310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856681310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784856689833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11185679584741592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3198333333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856706709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634652525186539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.3231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856723862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563578456640244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856741041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572417825460434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856741041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856741042, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856765300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761743068695068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856765301, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856765301, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784856782439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310821235179901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262144000, "lr": 1.3331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856799390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112926363945007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262799360, "lr": 1.3365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856816364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504800081253052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 263454720, "lr": 1.3398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856824829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856824830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856848631, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761879563331604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856848632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856848632, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784856857267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944179445505142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264110080, "lr": 1.3431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856874557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078421100974083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264765440, "lr": 1.3465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856891721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461512207984924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 265420800, "lr": 1.3498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856908837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037779226899147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266076160, "lr": 1.3531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856908838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856908838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856932411, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764398455619812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856932411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856932411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784856948889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402368009090424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266731520, "lr": 1.3565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856966405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709254443645477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 267386880, "lr": 1.359833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856983240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881530493497849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268042240, "lr": 1.3631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784856991808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784856991808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784857014975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763392925262451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784857014976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784857014976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784857023212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114377528429031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268697600, "lr": 1.3665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857040188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547736287117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 269352960, "lr": 1.3698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857056978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373465716838837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270008320, "lr": 1.3731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857073813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096193790435791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270663680, "lr": 1.3765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857073814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857073814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857097143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765759825706482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857097143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857097143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784857114217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077713966369629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271319040, "lr": 1.379833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857131371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123096942901611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271974400, "lr": 1.3831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857148193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090632826089859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 272629760, "lr": 1.3865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857156620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857156621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857180898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767032980918884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857180898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857180898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784857189329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178439855575562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273285120, "lr": 1.3898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857207071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095317453145981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273940480, "lr": 1.3931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857224047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11810500174760818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 274595840, "lr": 1.3965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857240748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609608680009842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275251200, "lr": 1.3998333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857240749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857240749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857264668, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768868803977966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857264668, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857264669, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784857281722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160030961036682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275906560, "lr": 1.4031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857298755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024048924446106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 276561920, "lr": 1.4065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857316102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11898262798786163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277217280, "lr": 1.4098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857324692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857324693, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857348885, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770905494689941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857348886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857348886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784857357600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310559511184692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277872640, "lr": 1.4131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857374561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993674397468567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 278528000, "lr": 1.4165000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857392329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769270151853561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279183360, "lr": 1.4198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857409548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982421785593033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279838720, "lr": 1.4231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857409549, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857409549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857433094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771967053413391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857433095, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857433095, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784857449790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112210899591446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 280494080, "lr": 1.4265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857466819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192137748003006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281149440, "lr": 1.4298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857483840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11591127514839172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281804800, "lr": 1.4331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857492359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857492359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857515916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772505283355713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857515916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857515916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784857524709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725043714046478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 282460160, "lr": 1.4365000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857542046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376588046550751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283115520, "lr": 1.4398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857559281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152923107147217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283770880, "lr": 1.4431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857576467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113654226064682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 284426240, "lr": 1.4464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857576468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857576468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857600384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772983908653259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857600384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857600384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784857617781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871952772140503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285081600, "lr": 1.4498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857634851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355166137218475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285736960, "lr": 1.4531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857651834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432386189699173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 286392320, "lr": 1.4564999999999997e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857660362, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857660363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857683214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772395014762878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857683214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857683214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784857691539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481152474880219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287047680, "lr": 1.4598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857708558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103422194719315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287703040, "lr": 1.4631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857725912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131301298737526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 288358400, "lr": 1.4664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857742861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430374532938004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289013760, "lr": 1.4698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857742862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857742862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857766671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774875164031982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857766672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857766672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784857783534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769511014223099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289669120, "lr": 1.4731666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857800407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571912676095963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290324480, "lr": 1.4764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857817977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120947659015656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290979840, "lr": 1.4798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857826444, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857826445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857849954, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775743007659912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857849955, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857849955, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784857858493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059933453798294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 291635200, "lr": 1.4831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857875348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906000435352325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292290560, "lr": 1.4864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857892307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506992042064667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292945920, "lr": 1.4898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857908953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564552992582321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 293601280, "lr": 1.4931666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857908954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857908954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857932578, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772654294967651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857932578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857932579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784857949101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821204632520676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294256640, "lr": 1.4964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857966130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389973223209381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294912000, "lr": 1.4998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857983418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783262550830841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 295567360, "lr": 1.5031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784857992090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784857992090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784858015707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773218154907227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784858015708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784858015708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784858024177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413098335266113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296222720, "lr": 1.5064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858040743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504911094903946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296878080, "lr": 1.5098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858058003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865291953086853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 297533440, "lr": 1.5131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858074918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949075222015381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298188800, "lr": 1.5164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858074919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858074919, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858098415, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773584723472595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858098415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858098415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784858115297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498364269733429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298844160, "lr": 1.5198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858132282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402709245681763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 299499520, "lr": 1.5231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858149233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379621386528015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300154880, "lr": 1.5265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858157865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858157865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858181881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772647738456726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858181882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858181882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784858190426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057762056589127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300810240, "lr": 1.5298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858207458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012814402580261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 301465600, "lr": 1.5331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858224585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167626291513443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302120960, "lr": 1.5364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858242076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736967623233795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302776320, "lr": 1.5398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858242077, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858242077, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858266414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772900462150574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858266414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858266414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784858283082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007805168628693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 303431680, "lr": 1.5431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858300639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813884437084198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304087040, "lr": 1.5465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858317878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412456840276718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304742400, "lr": 1.5498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858326481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858326482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858350385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775171399116516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858350385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858350385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784858358904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254718363285065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 305397760, "lr": 1.5531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858375724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770008713006973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306053120, "lr": 1.5564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858392576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551701486110687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306708480, "lr": 1.5598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858409300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581379383802414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 307363840, "lr": 1.5631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858409301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858409301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858432589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774600386619568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858432589, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858432589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784858449508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691876709461212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308019200, "lr": 1.5665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858466570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09893545508384705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308674560, "lr": 1.5698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858483568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470032691955566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309329920, "lr": 1.5731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858492174, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858492175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858515945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774308323860168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858515946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858515946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784858524589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804032534360886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309985280, "lr": 1.5764999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858541216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250061750411987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 310640640, "lr": 1.5798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858558925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975907742977142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311296000, "lr": 1.5831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858576172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083860769867897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311951360, "lr": 1.5865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858576172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858576173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858599290, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775616645812988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858599290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858599290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784858616318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665212571620941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 312606720, "lr": 1.5898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858633100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696311295032501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313262080, "lr": 1.5931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858650184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054968610405922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313917440, "lr": 1.5964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858658601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858658601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858681556, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778539657592773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858681557, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858681557, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784858690034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927486419677734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 314572800, "lr": 1.5998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858707046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045970544219017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315228160, "lr": 1.6031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858724261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377046465873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315883520, "lr": 1.6065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858741244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280053317546844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 316538880, "lr": 1.6098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858741244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858741245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858765131, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778280377388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858765132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858765132, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784858782525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808867961168289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317194240, "lr": 1.6131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858799362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975947976112366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317849600, "lr": 1.6165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858816654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359553039073944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 318504960, "lr": 1.6198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858825357, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858825358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858848916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779173851013184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858848916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858848916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784858857681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554653197526932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319160320, "lr": 1.6231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858874814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502948611974716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319815680, "lr": 1.6265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858892024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336571723222733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 320471040, "lr": 1.6298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858909252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078476682305336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321126400, "lr": 1.6331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858909252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858909253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858933129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781810760498047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858933129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858933129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784858950196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103670299053192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321781760, "lr": 1.6365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858967346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964231193065643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 322437120, "lr": 1.6398333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858984642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133769154548645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323092480, "lr": 1.6431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784858993274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784858993274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784859016659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783515453338623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784859016660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784859016660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784859025368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990244150161743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323747840, "lr": 1.6465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859042782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531838983297348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 324403200, "lr": 1.6498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859059639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677000880241394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325058560, "lr": 1.6531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859077643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453349888324738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325713920, "lr": 1.6565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859077644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859077644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859100757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784065008163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859100757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859100757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784859118167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101144403219223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 326369280, "lr": 1.6598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859135560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272601783275604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327024640, "lr": 1.6631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859153174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086471825838089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680000, "lr": 1.6665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859161855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859161856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859186683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786264419555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859186684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859186684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784859195119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473233997821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328335360, "lr": 1.6698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859212532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861527919769287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328990720, "lr": 1.6731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859229848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374745517969131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 329646080, "lr": 1.6765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859247322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903310030698776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330301440, "lr": 1.6798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859247323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859247324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859270985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786377668380737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859270985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859270985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784859288037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542695760726929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330956800, "lr": 1.6831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859305751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274464428424835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 331612160, "lr": 1.6865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859322844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371083766222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332267520, "lr": 1.6898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859331319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859331320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859353979, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785068154335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859353979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859353979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784859362535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764964669942856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332922880, "lr": 1.6931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859379349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453618317842484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 333578240, "lr": 1.6964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859396587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841890424489975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334233600, "lr": 1.6998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859413635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225097090005875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334888960, "lr": 1.7031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859413635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859413636, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859436364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784369587898254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859436364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859436364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784859453388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612834244966507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 335544320, "lr": 1.7064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859470796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674174129962921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336199680, "lr": 1.7098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859488040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928407311439514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336855040, "lr": 1.7131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859496771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859496772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859520048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783263921737671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859520049, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859520049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784859528857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081770271062851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 337510400, "lr": 1.7164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859545742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237932950258255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338165760, "lr": 1.7198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859563545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786841064691544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338821120, "lr": 1.7231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859580830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310006886720657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 339476480, "lr": 1.7265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859580831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859580832, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859604583, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784701585769653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859604583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859604584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784859621784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250856727361679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340131840, "lr": 1.7298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859638863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104228176176548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340787200, "lr": 1.7331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859656105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621675103902817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 341442560, "lr": 1.7364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859664736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859664736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859688268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786815762519836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859688269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859688269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784859696877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379502177238464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342097920, "lr": 1.7398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859713748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019432544708252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342753280, "lr": 1.7431666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859731092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690438747406006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 343408640, "lr": 1.7465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859748633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899271816015244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344064000, "lr": 1.7498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859748634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859748634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859772450, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785674333572388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859772450, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859772450, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784859789966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620206594467163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344719360, "lr": 1.7531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859807031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851576924324036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 345374720, "lr": 1.7564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859824159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110019147396088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346030080, "lr": 1.7598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859833382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859833383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859857104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786456942558289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859857104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859857104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784859865861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579589009284973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346685440, "lr": 1.7631666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859883386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349348932504654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347340800, "lr": 1.7665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859900653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765940696001053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347996160, "lr": 1.7698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859917657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155075043439865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 348651520, "lr": 1.7731666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859917726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859917726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859941931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787408828735352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859941931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859941931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784859959026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983335971832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349306880, "lr": 1.7764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859976134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794498026371002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349962240, "lr": 1.7798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784859993097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472961515188217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 350617600, "lr": 1.7831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860001626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860001627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860025253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787332534790039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860025254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860025254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784860033753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525750368833542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351272960, "lr": 1.7865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860050861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10128888487815857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351928320, "lr": 1.7898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860068095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012791320681572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 352583680, "lr": 1.7931666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860084613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542941838502884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353239040, "lr": 1.7964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860084683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860084684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860108448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789169549942017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860108448, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860108448, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784860126344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735345631837845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353894400, "lr": 1.7998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860144004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10624371469020844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 354549760, "lr": 1.8031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860161202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677920281887054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355205120, "lr": 1.8065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860169684, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860169685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860193680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791123986244202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860193680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860193681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784860202039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379913449287415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355860480, "lr": 1.8098333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860219294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220405459403992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 356515840, "lr": 1.8131666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860236662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308822244405746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357171200, "lr": 1.8164999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860253815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846558213233948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357826560, "lr": 1.8198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860253815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860253816, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860277647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791405320167542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860277647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860277647, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784860294984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192917287349701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 358481920, "lr": 1.8231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860312319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290251672267914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359137280, "lr": 1.8265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860329846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347205400466919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359792640, "lr": 1.8298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860338541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860338542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860362403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791821956634521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860362403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860362403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784860370674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623236745595932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 360448000, "lr": 1.8331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860388402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10879791527986526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361103360, "lr": 1.8365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860405942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917603224515915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361758720, "lr": 1.8398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860423196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110903799533844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 362414080, "lr": 1.8431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860423196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860423197, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860447356, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791388630867004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860447357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860447357, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784860464584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448715090751648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 363069440, "lr": 1.8465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860481761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000551283359528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 363724800, "lr": 1.8498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860499033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844696313142776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 364380160, "lr": 1.8531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860507594, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860507594, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860531205, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791720628738403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860531205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860531205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784860539830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908392071723938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 365035520, "lr": 1.8565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860556943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388638615608215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 365690880, "lr": 1.8598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860574166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484784096479416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 366346240, "lr": 1.8631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860591459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637783259153366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 367001600, "lr": 1.8665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860591460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860591460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860614860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793101668357849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860614860, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860614860, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784860632306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714592039585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 367656960, "lr": 1.869833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860649210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403244197368622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 368312320, "lr": 1.8731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860667443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117076575756073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 368967680, "lr": 1.8765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860676475, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860676476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860699524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795265316963196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860699525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860699525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784860708257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494135856628418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 369623040, "lr": 1.8798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860725637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097816526889801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 370278400, "lr": 1.8831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860742939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395252496004105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 370933760, "lr": 1.8865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860760490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151443421840668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 371589120, "lr": 1.889833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860760491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860760491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860784601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795580625534058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860784602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860784602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784860801969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923460125923157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 372244480, "lr": 1.8931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860819455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102035641670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 372899840, "lr": 1.8965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860837179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411987990140915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 373555200, "lr": 1.8998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860846092, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860846093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860869575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795700430870056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860869575, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860869575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784860878486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377333104610443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 374210560, "lr": 1.9031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860895986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457989573478699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 374865920, "lr": 1.9065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860913115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016236245632172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 375521280, "lr": 1.9098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860931569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318803578615189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 376176640, "lr": 1.9131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784860931570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784860931570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed412282151.log b/recommendation_v4/rcp_logs/gbs_32768/seed412282151.log new file mode 100644 index 000000000..4bd8c78e1 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed412282151.log @@ -0,0 +1,1427 @@ +:::MLLOG {"namespace": "", "time_ms": 1784881836574, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784881836574, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784881860034, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784881860035, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784881860035, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784881860035, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "seed", "value": 412282151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784881860036, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784881860037, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784882770650, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784882770650, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784882770973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784882908876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807998597621918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882924232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828760385513306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882938449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823862373828888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882945519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882945520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882994828, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009365081787109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882994829, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882994829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784883001986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807447254657745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883015986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380128264427185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883030796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381179839372635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883045116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381169855594635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883045116, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883045117, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883068237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501096785068512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883068237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883068237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883082511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375601887702942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883096826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377730369567871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883111261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758930563926697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883118171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883118171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883141781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012893080711365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883141781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883141782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883149195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798409700393677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883163557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796892762184143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883178275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801202178001404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883192748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748383522033691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883192749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883192749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883215949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015255212783813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883215949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883215949, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883230105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796773552894592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883244287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370914876461029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883258838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749809563159943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883265883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883265884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883289002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017967820167542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883289003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883289003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883295920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651180267333984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883310823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682329654693604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883324870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640326261520386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883339294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365259736776352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883339295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883339295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883363148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020853281021118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883363148, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883363148, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883377127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673245906829834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883391567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675953447818756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883406616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13589933514595032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883413863, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883413864, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883437620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024215579032898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883437621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883437621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883444916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691216707229614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883460568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585227727890015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883475389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362978219985962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883490822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678207993507385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883490822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883490823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883514424, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028579235076904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883514424, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883514424, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883529026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591539859771729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883544305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446034491062164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883558612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638511300086975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883565904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883565905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883589439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033006072044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883589440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883589440, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883596708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325722336769104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883611827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428348302841187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883626054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473141193389893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883640390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503065705299377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883640461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883640462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883664245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037540793418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883664246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883664246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883678475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350646197795868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883692772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445255160331726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883707099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350371241569519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883715027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883715028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883738754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042587518692017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883738755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883738755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883745710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561716675758362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883759972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471955060958862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883774583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181357085704803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883790198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450121879577637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883790199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883790199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883813849, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049453377723694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883813850, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883813850, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883828619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133632093667984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883843756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352507472038269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883859216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13488447666168213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883866726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883866727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883890637, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5059807896614075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883890638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883890638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883897926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363917589187622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883912858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346333801746368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883928147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429191708564758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883942918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320817619562149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883942980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883942980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883967031, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070451498031616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883967032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883967032, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883981710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293536007404327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883997239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122685253620148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884011810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321855664253235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884019119, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884019120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884042619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5081766843795776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884042620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884042620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884049916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849496841430664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884064716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112814724445343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884080133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12993282079696655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884094780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304386854171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884094781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884094782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884118328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5094400644302368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884118329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884118329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884132971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277746737003326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884148771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980109453201294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884165108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065706193447113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884172504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884172505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884195640, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.510814368724823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884195641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884195642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884203021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163718581199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884217719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351456344127655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884232880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332039088010788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884248327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356140375137329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884248327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884248328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884271366, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122842192649841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884271367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884271367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884286176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522332906723022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884300912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257700204849243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884315904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318942487239838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884323316, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884323317, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884346802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5137537121772766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884346802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884346803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884355312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641522824764252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884370751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527192175388336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884385950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617873191833496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884401409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647881150245667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884401410, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884401410, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884424861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.515275776386261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884424861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884424862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884440765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474561274051666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884456116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096243143081665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884471199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626818358898163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884478949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884478949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884502468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5167999267578125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884502468, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884502469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884510058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593316078186035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884525719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259334862232208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884540481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261431455612183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884555086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091184198856354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884555087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884555087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884578249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5183252692222595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884578249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884578249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884593174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103771209716797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884608687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832567632198334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884623573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295783817768097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884630872, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884630873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884653478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.520051121711731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884653479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884653479, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884660655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931504845619202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884675467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346992701292038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884690490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345359981060028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884706038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304713934659958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884706039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884706039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884728825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5220866799354553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884728826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884728826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884743611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12810611724853516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884758414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182762265205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884773446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354852318763733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884781035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884781036, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884804589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.524408221244812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884804590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884804590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884812139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663586974143982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884827644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336010843515396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884842510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206075131893158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884857118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099178671836853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884857118, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884857119, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884880037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5271634459495544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884880037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884880038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884895108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958845496177673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884910113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251513242721558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884925723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354474425315857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884933191, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884933191, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884956066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5302358865737915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884956067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884956067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884963453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195481896400452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884978682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331034779548645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884993870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349671334028244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885009128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459256291389465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885009129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885009129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885032219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5338658690452576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885032220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885032220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885047466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603444397449493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885063484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388536870479584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885079087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310499489307404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885086616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885086617, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885109784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5380253195762634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885109785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885109785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885117673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403258740901947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885133331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352105736732483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885148968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446646928787231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885164807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507713377475739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885164808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885164809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885187407, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5424267649650574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885187408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885187408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885202595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523012399673462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885218015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332034170627594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885233520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218024373054504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885241849, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885241850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885265743, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5473649501800537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885265744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885265744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885273529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249912858009338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885288816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073000311851501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885303981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135860413312912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885319439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221409916877747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885319440, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885319440, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885342651, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5533077716827393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885342652, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885342652, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885358175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453739881515503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885374106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004320859909058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885389195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169467449188232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885396671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885396671, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885419726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5610353946685791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885419726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885419727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885427122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321503072977066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885442358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046523928642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885457881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287129998207092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885473830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242560625076294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885473831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885473831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885497242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5708088278770447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885497243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885497243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885512574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173486292362213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885527827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780660390853882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885542866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337345540523529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885550790, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885550791, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885574488, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5811532735824585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885574488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885574488, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885582246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13439635932445526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885597641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264544427394867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885613505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12648701667785645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885628902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324400007724762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885628902, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885628903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885651843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5924150347709656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885651844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885651844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885666973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871401011943817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885682073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12798070907592773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885697764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736761569976807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885705344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885705344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885728238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6008750200271606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885728239, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885728239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885736025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963394820690155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885751743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12685397267341614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885766960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948161363601685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885782203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753652036190033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885782204, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885782204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885805197, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6110828518867493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885805198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885805198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885820226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1260138601064682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885835767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289222687482834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885851061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188913464546204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885858731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885858732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885881960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6200058460235596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885881961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885881961, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885889996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015779852867126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885905873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12530440092086792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885921650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034780323505402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885937173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12510843575000763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885937173, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885937174, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885960867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.629524827003479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885960868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885960868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885976807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760215997695923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885992585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288541704416275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886008760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691180408000946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886016630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886016630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886039837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6391634345054626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886039838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886039838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886047769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607929110527039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886063073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577877938747406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886078647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12622126936912537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886094249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252909004688263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886094250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886094250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886118028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.646886944770813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886118029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886118029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886133949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920290440320969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886149503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11874064058065414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886164710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11666086316108704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886172343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886172344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886196321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6496631503105164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886196322, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886196322, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886203808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870196461677551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886219226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580918729305267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886234737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12047069519758224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886250047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11777995526790619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886250047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886250047, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886274474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6729874610900879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886274474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886274475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886290556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940652877092361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886305900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047067493200302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886321310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11647825688123703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886328874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886328875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886352724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6904646754264832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886352725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886352725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886360216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731501668691635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886375933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11719302088022232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886391692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204637587070465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886406958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569606512784958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886407027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886407027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886431100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7187668681144714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886431101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886431101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886446681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451415717601776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886462007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461204290390015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886477597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838174819946289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886485241, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886485242, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886508251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7376513481140137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886508252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886508252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886515991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200104653835297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886531720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328410357236862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886547332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112077534198761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886563168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381107568740845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886563168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886563169, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886586776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7541411519050598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886586776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886586777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886602277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11276717483997345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886617534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182033270597458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886632922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556382477283478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886640493, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886640494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886664140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7577847838401794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886664140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886664141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886671695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057494729757309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886687385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135985404253006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886702924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756121575832367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886718563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232875287532806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886718564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886718565, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886742807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.761486291885376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886742808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886742808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886758383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483544856309891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886773784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261174082756042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886789171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072814464569092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886796629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886796630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886819923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7631018757820129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886819924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886819924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886827701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106744110584259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886843385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278600990772247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886859375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238733679056168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886874810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728377103805542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886874811, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886874811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886898512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7646206021308899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886898512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886898513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886915399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565229296684265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886931040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194465160369873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886946742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721863597631454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886954502, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886954503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886977854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652996182441711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886977854, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886977854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784886985558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605584621429443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887001503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11522561311721802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887017261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281131207942963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887032837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350332200527191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887032838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887032838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887056276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656848430633545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887056277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887056277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887072381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834351718425751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887088134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175394132733345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887104235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069384962320328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887112301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887112303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887135144, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7660859227180481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887135145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887135145, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887143176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623311787843704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887159389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538176983594894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887175399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111554279923439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887191833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459825396537781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887191833, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887191834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887214570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662698030471802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887214570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887214570, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887229779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683605074882507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887245382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601920634508133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887260790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600610077381134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887268651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887268652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887291904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7663646936416626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887291905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887291905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887299745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400446504354477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887315781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147905141115189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887331618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775072127580643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887347726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129511147737503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887347727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887347727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887371622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766907811164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887371623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887371623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887387764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723884403705597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887403306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649074614048004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887419059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242845118045807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887426750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887426750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887449883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671055793762207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887449884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887449884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887457809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027032881975174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887473615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10130460560321808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887489824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181332170963287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887505507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480780899524689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887505508, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887505508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887529178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678101062774658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887529179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887529179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887545338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658878833055496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887561206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749409347772598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887576710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695990920066833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887584796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887584797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887608133, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768152117729187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887608134, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887608134, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887616062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724715888500214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887631583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000676229596138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887647358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787332057952881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887663171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470045357942581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887663172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887663172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887687333, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691392302513123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887687334, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887687334, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887702754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770837962627411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887719105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080397367477417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887734988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476988554000854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887742786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887742786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887765794, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692858576774597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887765794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887765794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887773554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109261155128479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887789413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724271088838577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887805095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119142398238182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887821142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790212452411652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887821142, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887821143, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887844164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697048783302307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887844164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887844165, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887860356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544456541538239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887876064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269701480865479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887892398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085181012749672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887900528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887900528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887923227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699686288833618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887923228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887923228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887930944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440301895141602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887946263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996471643447876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887961915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429009586572647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887977319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113195866346359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887977319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784887977320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888000245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703400254249573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888000246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888000246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888015840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675999522209167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888031895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888203233480453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888047697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815788060426712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888055446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888055446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888079062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708685398101807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888079063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888079063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888087737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026258021593094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888104055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482395231723785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888120268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256112903356552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888136530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387111246585846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888136531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888136531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888159396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714524865150452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888159397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888159397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888175350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051137000322342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888191738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383944749832153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888208084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094431579113007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888216163, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888216163, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888239996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715817093849182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888239996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888239997, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888247941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110675185918808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888264501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952961444854736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888280592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302266269922256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888296741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151298880577087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888296741, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888296742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888319548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717637419700623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888319549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888319549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888335765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046922206878662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888352209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112847238779068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888368522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714545637369156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888376611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888376612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888400135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723289728164673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888400136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888400136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888407945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468864977359772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888424372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795850306749344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888440307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158606618642807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888455974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731766372919083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888455974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888455975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888479120, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714527249336243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888479121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888479121, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888494714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515391081571579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888510219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674387216567993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888526259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589493811130524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888534374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888534374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888558213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721667289733887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888558214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888558214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888566165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086978018283844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888581805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10337284207344055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888598289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471700131893158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888614567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748215764760971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888614568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888614568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888637206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729381918907166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888637206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888637206, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888652684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006227359175682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888668595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405139625072479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888684234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346541553735733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888692000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888692000, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888715249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729799747467041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888715250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888715250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888723212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432588309049606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888739502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318921506404877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888755297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437897384166718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888771447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358002036809921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888771448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888771449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888795740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733559608459473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888795741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888795741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888811528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719603300094604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888827149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773928463459015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888842752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171644389629364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888850519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888850519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888875097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773585319519043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888875097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888875097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888882813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142611712217331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888898368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042697936296463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888914276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647735744714737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888930398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317666828632355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888930398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888930399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888954316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738311290740967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888954317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888954317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784888969837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767389088869095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888986921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407574474811554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889002691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062961220741272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889010398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889010398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889034276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741561532020569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889034277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889034277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889042098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396243631839752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889057884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864794254302979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889073781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333497077226639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889090021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691781342029572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889090022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889090022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889113431, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739772200584412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889113432, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889113432, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889129703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249138623476028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889145454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708852112293243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889161702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099308431148529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889169642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889169642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889193743, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745820879936218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889193744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889193744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889201937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218194663524628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889217518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745362937450409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889233343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030845195055008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889249294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029382199048996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889249294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889249294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889273113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748270034790039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889273114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889273114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889288575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542712360620499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889304798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553093254566193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889320905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196473240852356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889328931, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889328931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889352320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746824622154236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889352320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889352321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889360105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701341927051544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889376384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258228123188019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889393081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984642803668976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889409357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281634867191315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889409358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889409358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889432830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752846479415894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889432831, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889432831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889449160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888948291540146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889465466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408251523971558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889481746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205873638391495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889489903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889489904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889513835, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752264738082886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889513836, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889513836, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889522057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272973567247391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889538419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211259663105011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889555021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772246867418289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889571461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093437671661377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889571461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889571461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889594877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775435209274292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889594878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889594878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889610746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11426657438278198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889626830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900958627462387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889642977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160405725240707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889651218, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889651219, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889675034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758574485778809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889675035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889675035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784889683443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142663657665253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889700116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403945297002792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889716284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027885228395462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889732864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821132361888885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889732865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889732865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889756019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755417823791504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889756020, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889756020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784889771842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392211377620697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889787279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993409365415573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889803071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019909679889679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889810962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889810963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889833999, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758267521858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889834000, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889834000, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784889841864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997801423072815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889857438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138490051031113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889873226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034000962972641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889889322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056935042142868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889889323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889889323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889912475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759043574333191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889912475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889912475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784889928026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461531579494476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889944453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991238802671432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889960558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648208856582642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889968551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889968551, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889991584, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760001420974731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889991584, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889991584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784889999369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180509835481644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890015446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575634241104126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890031368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007687896490097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890047203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166770219802856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890047203, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890047204, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890070904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761595249176025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890070905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890070905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890087140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224998742341995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890103716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349644720554352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890119704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181119292974472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890128487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890128488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890153319, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766220569610596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890153319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890153320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890161242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438612848520279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890177329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772015154361725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890193203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294751822948456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890209192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070635378360748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890209193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890209193, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890233077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762840390205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890233078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890233078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890249007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617813467979431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890264804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205268859863281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890280910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202352702617645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890289085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890289085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890312587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766115665435791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890312588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890312588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890320642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169992387294769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890336499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978461802005768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890352922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802322626113892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890369307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087338849902153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890369308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890369308, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890393078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767050862312317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890393079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890393079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890409241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323735326528549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890432420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049700453877449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890448573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085123032331467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890456728, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890457895, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890481386, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770466804504395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890481386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890481387, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890492652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558268427848816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890509062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445626825094223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890525401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702858120203018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890541982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957591235637665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890541982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890541982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890565744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767850160598755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890565745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890565745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890581807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087769865989685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890598583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924778878688812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890615275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792893916368484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890623357, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890623357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890646972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775548100471497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890646972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890646972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784890654867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261694878339767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890670978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995608568191528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890693324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072986051440239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890709377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688146948814392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890709378, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890709379, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890732744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777417778968811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890732744, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890732745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784890750525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705672204494476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890766511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548437386751175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890790210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085185408592224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890797945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890797946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890822312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773594260215759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890822312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890822312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784890830296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878045856952667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890847283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959143936634064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890864101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109735071659088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890880755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164698004722595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890880756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890880756, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890904773, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778894305229187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890904774, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890904774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784890921555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401958763599396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890940764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351203173398972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890957253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116446554660797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890965610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890965611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890990109, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778226137161255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890990109, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890990109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784890998264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877678543329239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891015262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281182616949081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891031583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136709451675415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891048546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194048821926117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.2131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891048546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891050528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891073547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778149425983429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891073547, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891073547, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784891089936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11748066544532776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891106115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387547850608826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891129173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380765587091446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891137423, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891137423, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891160969, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782598733901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891160969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891160970, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784891168887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148531436920166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891194614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150275245308876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891211188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443700641393661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891227385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791713535785675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891227386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891227386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891251367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785623669624329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891251367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891251367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784891268527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973426699638367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891294798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155354306101799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.2431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891315300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144709661602974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891323850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891323850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891347579, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784359455108643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891347579, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891347580, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784891355728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052628606557846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891380032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123395711183548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891396553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779352486133575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891414725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790731757879257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891414725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891414726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891438166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787181735038757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891438167, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891438168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784891454478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133267804980278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.2631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891474945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972362756729126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891491473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825686901807785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891499586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891499587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891522987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782536745071411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891522987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891522987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784891530774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615513473749161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891553868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496032983064651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891571239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424234718084335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891587568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842503607273102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.2831666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891587568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891587569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891611158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787237167358398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891611158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891611158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784891627833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064860224723816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891648772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539144277572632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891664893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778208076953888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891673448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891673449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891697110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790219187736511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891697111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891697111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784891705209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714731365442276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891721517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707167536020279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891744177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914561152458191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.3031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891760662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668477416038513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891760662, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891760662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891784336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791630625724792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891784337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891784337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784891805377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662214457988739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891821369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520495474338531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891839377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398474335670471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891853974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891853975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891877759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792909741401672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891877760, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891877760, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784891893718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158297955989838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3198333333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891909974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542181134223938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.3231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891926540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575206577777863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891942921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583299398422241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891942922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891942922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891966487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779312014579773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891966487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891966487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784891988425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351482033729553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262144000, "lr": 1.3331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892004494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145476460456848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262799360, "lr": 1.3365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892025278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555120557546616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 263454720, "lr": 1.3398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892047222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892047223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892070552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795237898826599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892070553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892070554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784892083479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941708832979202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264110080, "lr": 1.3431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892100066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082533597946167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264765440, "lr": 1.3465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892120083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494057089090347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 265420800, "lr": 1.3498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892143277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395629703998566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266076160, "lr": 1.3531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892143278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892143278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892167260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797556519508362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892167260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892167261, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784892183269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440080612897873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266731520, "lr": 1.3565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892204871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722469538450241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 267386880, "lr": 1.359833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892221189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904078930616379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268042240, "lr": 1.3631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892229494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892229495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892252868, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795620560646057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892252868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892252869, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784892260768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116981506347656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268697600, "lr": 1.3665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892284226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530374199151993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 269352960, "lr": 1.3698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892301685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369445383548737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270008320, "lr": 1.3731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892318014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945510864257812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270663680, "lr": 1.3765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892318015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892318015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892342946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7801367044448853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892342946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892342947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784892359486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088544130325317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271319040, "lr": 1.379833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892376064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114522069692612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271974400, "lr": 1.3831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892392302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092117503285408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 272629760, "lr": 1.3865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892400486, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892400486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892425621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798935770988464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892425622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892425622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784892433749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127183586359024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273285120, "lr": 1.3898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892450859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063509434461594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273940480, "lr": 1.3931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892467122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11858926713466644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 274595840, "lr": 1.3965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892483089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11748874932527542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275251200, "lr": 1.3998333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892483090, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892483090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892506621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805247902870178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892506622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892506622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784892523124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163142323493958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275906560, "lr": 1.4031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892539519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092260479927063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 276561920, "lr": 1.4065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892556283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183691918849945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277217280, "lr": 1.4098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892564542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892564543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892587672, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806130051612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892587672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892587672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784892596134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328991502523422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277872640, "lr": 1.4131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892612631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968223214149475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 278528000, "lr": 1.4165000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892629874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832235753536224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279183360, "lr": 1.4198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892646515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964210331439972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279838720, "lr": 1.4231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892646515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892646516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892669931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808818221092224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892669931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892669932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784892686084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131740361452103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 280494080, "lr": 1.4265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892702481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222444474697113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281149440, "lr": 1.4298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892718782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529958248138428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281804800, "lr": 1.4331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892726904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892726904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892750145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807187438011169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892750146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892750146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784892758731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731026530265808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 282460160, "lr": 1.4365000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892775385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370497941970825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283115520, "lr": 1.4398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892792117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116315945982933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283770880, "lr": 1.4431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892808677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120506376028061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 284426240, "lr": 1.4464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892808678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892808678, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892833832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78078693151474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892833833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892833833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784892850547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855686664581299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285081600, "lr": 1.4498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892867079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371850967407227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285736960, "lr": 1.4531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892883563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424928903579712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 286392320, "lr": 1.4564999999999997e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892891861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892891862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892915509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78091961145401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892915510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892915510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784892923596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450488865375519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287047680, "lr": 1.4598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892940089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068125069141388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287703040, "lr": 1.4631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892957067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296287178993225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 288358400, "lr": 1.4664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892973949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344553530216217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289013760, "lr": 1.4698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784892973949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784892973949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784892997627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812180519104004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784892997628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784892997628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784893014130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764484107494354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289669120, "lr": 1.4731666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893030675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561063140630722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290324480, "lr": 1.4764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893047816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119298100471497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290979840, "lr": 1.4798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893055975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893055975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893079533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813050746917725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893079534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893079534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784893087801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596254467964172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 291635200, "lr": 1.4831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893104161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911301523447037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292290560, "lr": 1.4864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893120689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537146776914597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292945920, "lr": 1.4898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893136979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552927851676941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 293601280, "lr": 1.4931666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893136980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893136980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893160928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812530994415283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893160928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893160929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784893177265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083410456776619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294256640, "lr": 1.4964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893193995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415935516357422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294912000, "lr": 1.4998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893211003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831423103809357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 295567360, "lr": 1.5031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893219479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893219480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893243883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816811800003052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893243883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893243884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784893252101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419677942991257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296222720, "lr": 1.5064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893268264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508085042238235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296878080, "lr": 1.5098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893285461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849079489707947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 297533440, "lr": 1.5131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893302026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993081331253052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298188800, "lr": 1.5164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893302027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893302028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893326083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818453907966614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893326084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893326084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784893342620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521847009658813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298844160, "lr": 1.5198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893366340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350354015827179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 299499520, "lr": 1.5231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893383535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401301831007004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300154880, "lr": 1.5265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893391747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893391748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893415720, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815906405448914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893415721, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893415721, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784893423825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037202924489975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300810240, "lr": 1.5298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893446814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011391341686249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 301465600, "lr": 1.5331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893463165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127050429582596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302120960, "lr": 1.5364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893489184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707957297563553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302776320, "lr": 1.5398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893489185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893489187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893513434, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814986109733582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893513434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893513434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784893529478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999277234077454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 303431680, "lr": 1.5431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893546275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750923305749893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304087040, "lr": 1.5465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893562986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381879657506943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304742400, "lr": 1.5498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893572078, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893572079, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893595753, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7821285724639893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893595753, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893595753, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784893616245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237764567136765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 305397760, "lr": 1.5531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893637602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762208700180054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306053120, "lr": 1.5564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893653878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559121519327164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306708480, "lr": 1.5598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893676858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059480607509613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 307363840, "lr": 1.5631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893676859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893676859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893700608, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820316553115845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893700608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893700609, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784893716801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679879039525986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308019200, "lr": 1.5665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893739921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924949705600739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308674560, "lr": 1.5698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893757319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427863895893097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309329920, "lr": 1.5731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893771515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893771516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893795184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820866107940674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893795184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893795184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784893803383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791271924972534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309985280, "lr": 1.5764999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893819281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196676105260849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 310640640, "lr": 1.5798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893836453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091044545173645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311296000, "lr": 1.5831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893853142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756444931030273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311951360, "lr": 1.5865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893853143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893855932, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893878695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7827751636505127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893878696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893878696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784893898787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556105524301529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 312606720, "lr": 1.5898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893919982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632667690515518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313262080, "lr": 1.5931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893936447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531498491764069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313917440, "lr": 1.5964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784893952515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893952516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893976436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7830345034599304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893976436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893976436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784893984595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918641090393066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 314572800, "lr": 1.5998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894000850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450717806816101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315228160, "lr": 1.6031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894028627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411831736564636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315883520, "lr": 1.6065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894045003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365171730518341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 316538880, "lr": 1.6098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894045004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894045005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894068781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829300761222839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894068781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894068781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784894085677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770942270755768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317194240, "lr": 1.6131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894102035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091964840888977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317849600, "lr": 1.6165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894119175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342395097017288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 318504960, "lr": 1.6198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894127543, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894127544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894151346, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831977605819702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894151346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894151346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784894159750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460433155298233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319160320, "lr": 1.6231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894176170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474401712417603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319815680, "lr": 1.6265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894192701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296282708644867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 320471040, "lr": 1.6298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894210419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769301652908325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321126400, "lr": 1.6331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894210420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894210420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894234573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831026315689087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894234573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894234574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784894251068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067172884941101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321781760, "lr": 1.6365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894267752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913459211587906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 322437120, "lr": 1.6398333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894284538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297105997800827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323092480, "lr": 1.6431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894292847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894292848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894316682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7838637828826904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894316683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894316683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784894325137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910353064537048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323747840, "lr": 1.6465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894342045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494019091129303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 324403200, "lr": 1.6498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894358424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055947095155716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325058560, "lr": 1.6531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894375834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418972909450531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325713920, "lr": 1.6565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894375835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894375835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894399436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837482690811157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894399436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894399436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784894416274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097169816493988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 326369280, "lr": 1.6598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894432783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206197738647461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327024640, "lr": 1.6631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894449819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756146162748337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680000, "lr": 1.6665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894458198, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894458198, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894481896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839120626449585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894481897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894481897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784894490135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348499357700348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328335360, "lr": 1.6698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894506896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823830962181091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328990720, "lr": 1.6731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894523864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373060941696167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 329646080, "lr": 1.6765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894540860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907566547393799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330301440, "lr": 1.6798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894540860, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894540861, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894564170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845208644866943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894564170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894564170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784894580803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450023204088211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330956800, "lr": 1.6831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894598155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260047554969788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 331612160, "lr": 1.6865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894614654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361038893461227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332267520, "lr": 1.6898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894622925, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894622926, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894647066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840016484260559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894647066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894647066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784894655429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751772671937943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332922880, "lr": 1.6931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894671684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047401949763298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 333578240, "lr": 1.6964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894694274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799053311347961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334233600, "lr": 1.6998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894710727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173635929822922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334888960, "lr": 1.7031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894710728, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894710728, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894734153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845655679702759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894734153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894734153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784894750289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583449900150299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 335544320, "lr": 1.7064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894771788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067437157034874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336199680, "lr": 1.7098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894788277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887971520423889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336855040, "lr": 1.7131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894796656, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894796657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894820733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840965390205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894820733, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894820733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784894829122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034190863370895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 337510400, "lr": 1.7164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894845317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226625949144363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338165760, "lr": 1.7198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894862490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781531780958176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338821120, "lr": 1.7231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894886494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244381427764893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 339476480, "lr": 1.7265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894886495, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894886495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894909050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843897938728333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894909051, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894909051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784894925476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224098712205887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340131840, "lr": 1.7298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894951904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383173078298569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340787200, "lr": 1.7331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894968334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637040436267853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 341442560, "lr": 1.7364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784894979345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784894979346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784895002977, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7848254442214966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784895002977, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784895002978, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784895011384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392780601978302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342097920, "lr": 1.7398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895027675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137007385492325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342753280, "lr": 1.7431666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895052992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645703971385956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 343408640, "lr": 1.7465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895076821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085062026977539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344064000, "lr": 1.7498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895076822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895076822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895100486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7848402857780457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895100486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895100486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784895117383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616272687911987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344719360, "lr": 1.7531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895136003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741312801837921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 345374720, "lr": 1.7564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895152358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100556880235672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346030080, "lr": 1.7598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895161137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895161137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895184555, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852123379707336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895184555, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895184556, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784895196088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533897578716278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346685440, "lr": 1.7631666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895213085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342331975698471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347340800, "lr": 1.7665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895237115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680028051137924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347996160, "lr": 1.7698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895253681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101582854986191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 348651520, "lr": 1.7731666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895260087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895260087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895282886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7849647998809814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895282886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895282886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784895299399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905459523200989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349306880, "lr": 1.7764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895316052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746116191148758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349962240, "lr": 1.7798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895332587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431288182735443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 350617600, "lr": 1.7831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895340962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895340964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895364659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7851539850234985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895364659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895364659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784895372993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486442595720291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351272960, "lr": 1.7865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895389690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076399892568588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351928320, "lr": 1.7898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895406512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032033920288086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 352583680, "lr": 1.7931666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895422632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564962029457092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353239040, "lr": 1.7964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895422708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895422709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895446994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852594256401062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895446994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895446994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784895464592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632521659135818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353894400, "lr": 1.7998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895481701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607919096946716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 354549760, "lr": 1.8031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895498238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602600872516632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355205120, "lr": 1.8065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895506415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895506415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895530083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7856638431549072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895530084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895530084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784895538135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370985418558121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355860480, "lr": 1.8098333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895554716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173111408948898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 356515840, "lr": 1.8131666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895571084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026175320148468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357171200, "lr": 1.8164999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895594843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742859542369843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357826560, "lr": 1.8198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895594844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895594844, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895617619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7857645153999329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895617619, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895617620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784895634381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120522975921631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 358481920, "lr": 1.8231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895651118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234185308218002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359137280, "lr": 1.8265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895668008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315757989883423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359792640, "lr": 1.8298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895676390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895676391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895700249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7861076593399048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895700249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895700250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784895708190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542351007461548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 360448000, "lr": 1.8331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895725119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084589809179306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361103360, "lr": 1.8365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895742095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876689851284027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361758720, "lr": 1.8398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895758764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057984828948975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 362414080, "lr": 1.8431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895758764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895758765, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895782371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7862517237663269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895782371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895782372, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784895798943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429901629686356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 363069440, "lr": 1.8465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895815500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907542705535889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 363724800, "lr": 1.8498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895832093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790199786424637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 364380160, "lr": 1.8531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895840249, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895840249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895863719, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7864620685577393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895863719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895863719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 364707840, "epoch_num": 0.15920298609770536}} +:::MLLOG {"namespace": "", "time_ms": 1784895872060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829105973243713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 365035520, "lr": 1.8565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895888779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081413179636002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 365690880, "lr": 1.8598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895905708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451792180538177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 366346240, "lr": 1.8631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895922603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537559539079666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 367001600, "lr": 1.8665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895922603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895922604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895946715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7865036129951477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895946715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895946715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 367001600, "epoch_num": 0.1602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784895963750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692057758569717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 367656960, "lr": 1.869833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895980268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038617417216301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 368312320, "lr": 1.8731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784895998072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203882098197937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 368967680, "lr": 1.8765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896006715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896006716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896030632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7866503596305847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896030633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896030633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 369295360, "epoch_num": 0.16120553938195323}} +:::MLLOG {"namespace": "", "time_ms": 1784896039097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553560197353363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 369623040, "lr": 1.8798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896055810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953158885240555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 370278400, "lr": 1.8831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896072447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304080486297607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 370933760, "lr": 1.8865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896089339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092185229063034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 371589120, "lr": 1.889833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896089340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896089340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896113116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.786654531955719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896113117, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896113117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 371589120, "epoch_num": 0.16220681602407716}} +:::MLLOG {"namespace": "", "time_ms": 1784896129885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873869061470032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 372244480, "lr": 1.8931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896146654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083774268627167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 372899840, "lr": 1.8965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896163473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291994899511337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 373555200, "lr": 1.8998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896171878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896171878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896195678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7868607044219971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896195678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896195679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 373882880, "epoch_num": 0.1632080926662011}} +:::MLLOG {"namespace": "", "time_ms": 1784896204149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132204607129097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 374210560, "lr": 1.9031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896220997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438196361064911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 374865920, "lr": 1.9065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896237640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985841602087021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 375521280, "lr": 1.9098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896255470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123722568154335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 376176640, "lr": 1.9131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896255470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896255471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896279291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7867296934127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896279292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896279292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 376176640, "epoch_num": 0.16420936930832503}} +:::MLLOG {"namespace": "", "time_ms": 1784896296208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039707064628601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 376832000, "lr": 1.9165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896313252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585296899080276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 377487360, "lr": 1.9198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896330128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862220078706741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 378142720, "lr": 1.9231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896338698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896338699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896362530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7875293493270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896362531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896362531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 378470400, "epoch_num": 0.16521064595044896}} +:::MLLOG {"namespace": "", "time_ms": 1784896370867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947298258543015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 378798080, "lr": 1.9265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896387801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659389734268188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 379453440, "lr": 1.929833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896405201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182478070259094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 380108800, "lr": 1.9331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896422358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455091834068298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 380764160, "lr": 1.9365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784896422359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 380764160, "epoch_num": 0.1662119225925729}} +:::MLLOG {"namespace": "", "time_ms": 1784896422359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 380764160, "epoch_num": 0.1662119225925729}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed430583489.log b/recommendation_v4/rcp_logs/gbs_32768/seed430583489.log new file mode 100644 index 000000000..a1d96707e --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed430583489.log @@ -0,0 +1,1333 @@ +:::MLLOG {"namespace": "", "time_ms": 1784872301635, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784872301636, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784872325188, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "seed", "value": 430583489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784872325189, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784872325190, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784872325190, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784872325191, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784873099877, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784873099878, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784873100180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784873381678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909201323986053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784873397519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892775774002075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784873412470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894310593605042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784873419914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873419914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873472090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016211867332458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873472091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873472091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784873479617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894794881343842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873494196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903319835662842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873509455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387622058391571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873524358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880331814289093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873524358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873524359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873550549, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017668008804321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873550549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873550549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784873565483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882702589035034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873580477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387886255979538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873595929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869570195674896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873603097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873603097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873629206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019413232803345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873629207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873629207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784873637115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385650932788849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873652183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865023851394653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873667542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857436180114746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873682984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836699724197388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873682985, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873682985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873709291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021404027938843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873709292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873709292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784873724386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842740654945374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873739710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379508376121521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873755395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824157416820526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873762995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873762996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873789491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023781061172485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873789492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873789492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784873796759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759472966194153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873812706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766053318977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873827683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741396367549896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873843161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733963668346405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873843162, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873843163, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873871246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026318430900574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873871247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873871247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784873886181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737431168556213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873901308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374024599790573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873917732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677534461021423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873925304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873925304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873952360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028992891311646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873952361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873952361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784873960045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373516321182251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873976658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646158576011658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784873992178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682430982589722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784874009073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719674944877625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784874009074, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874009074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874035858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032258033752441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874035859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874035859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784874051159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136397123336792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784874068002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519124686717987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784874083337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669180870056152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874090977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874090977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874118185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035300850868225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874118186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874118186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784874125767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347268104553223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874142046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495463132858276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874157220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350741684436798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874172297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534510135650635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874172369, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874172369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874201807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038523077964783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874201808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874201808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784874217518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393738865852356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874232793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472744822502136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874248035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532942533493042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874256855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874256856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874284366, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041912198066711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874284367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874284367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784874292020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576596975326538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874307296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494935631752014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874322814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322031319141388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874339797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467763364315033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874339797, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874339798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874365531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046853423118591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874365531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874365531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784874381315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384094834327698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874397020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135206937789917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874413655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350545585155487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874421668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874421669, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874448105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055559277534485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874448106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874448106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784874455711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631513714790344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874471306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350194692611694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874488005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435465097427368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874503827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321951150894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874503894, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874503894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874530972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064919590950012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874530972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874530972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784874546272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284419476985931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874563180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114963471889496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874579260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312779366970062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874587339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874587340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874613817, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074946284294128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874613818, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874613818, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784874621557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836759328842163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874636995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103565573692322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874653717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998874485492706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874669510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13041874766349792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874669511, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874669511, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874695403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5087280869483948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874695404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874695404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784874711038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327371597290039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874726532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974074482917786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874743301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072185218334198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874751059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874751060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874776830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100922584533691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874776830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874776831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784874784707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152679800987244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874800485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351388543844223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874816193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307194411754608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874833250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563579320907593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874833251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874833251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874859516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5116199851036072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874859516, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874859517, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784874875490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516424596309662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874891289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325656920671463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874906893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323219120502472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874914658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874914658, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874941674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.513173520565033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874941675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874941675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784874950629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363818198442459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874967218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523449003696442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874983696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362694650888443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874999897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640615344047546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874999898, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784874999898, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784875027393, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5146539211273193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784875027394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784875027394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784875044835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481596112251282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875061430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084128499031067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875077516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618192076683044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875085614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875085615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875112993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.51615309715271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875112994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875112994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784875120955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596248626708984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875138349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246837258338928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875154592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261884450912476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875170785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307477355003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875170786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875170786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875197116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5175656676292419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875197116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875197117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784875212834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103370368480682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875229662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815760612487793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875246283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12957757711410522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875254246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875254246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875280443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5191511511802673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875280444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875280444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784875288181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12937575578689575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875303905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473454117774963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875319606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457587361335754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875336588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304568499326706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875336588, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875336589, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875363376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5210553407669067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875363376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875363377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784875379492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812873721122742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875395602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195113837718964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875411766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556939363479614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875419728, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875419729, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875446190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5231814980506897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875446190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875446190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784875454060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652411103248596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875471086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335722953081131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875487190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214372098445892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875503133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309097707271576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875503133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875503134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875529715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.525613009929657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875529715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875529715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784875545779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960535287857056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875561468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324344426393509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875578335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353551149368286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875586711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875586712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875612337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.528427004814148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875612338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875612338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784875620286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194602727890015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875636762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334417760372162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875653003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498324155807495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875668940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451890647411346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875668941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875668941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875695923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5316962003707886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875695924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875695924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784875712085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359575092792511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875729312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395200669765472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875746043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313175737857819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875754112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875754113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875780866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5352805256843567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875780866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875780867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784875789289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14031895995140076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875805462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351775974035263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875822116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343766748905182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875838811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351994276046753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875838811, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875838812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875865314, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5390453934669495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875865314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875865315, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784875881851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352241188287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875898352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323475420475006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875914489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242769241333008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875923685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875923686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875951273, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5433268547058105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875951274, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875951274, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784875959490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241755962371826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875976045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080596923828125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875992399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135768324136734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876008787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225997984409332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876008788, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876008788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876035002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5484570860862732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876035003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876035003, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784876051222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463056087493896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876068458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301889717578888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876085032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318211853504181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876093161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876093161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876119487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5553075075149536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876119487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876119487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784876127589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13213148713111877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876144111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059677183628082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876160318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295882940292358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876177098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250571489334106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876177099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876177099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876203696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5642054677009583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876203696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876203697, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784876220025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198570907115936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876236322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282147765159607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876252287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339353770017624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876260496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876260497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876287005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5738388299942017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876287005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876287005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784876295313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451720774173737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876311391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12665829062461853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876328432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266464740037918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876344910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279074430465698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876344911, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876344911, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876370412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5861564874649048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876370413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876370413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784876386651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907230854034424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876402918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12835894525051117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876419515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12784025073051453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876427262, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876427262, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876453591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5952786207199097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876453592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876453592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784876461633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975049018859863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876478755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272677481174469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876495310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982437014579773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876511780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12810683250427246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876511780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876511781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876537901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6071932315826416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876537901, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876537901, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784876554108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12639012932777405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876570924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293286681175232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876586949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241958618164062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876594924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876594924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876621499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6194169521331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876621499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876621500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784876630477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077670335769653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876647555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261427402496338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876664476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130892813205719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876681169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12603875994682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876681169, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876681170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876707326, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6325674653053284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876707326, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876707327, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784876724291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12845271825790405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876740662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954001128673553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876758213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751400470733643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876766764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876766765, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876793464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.647455632686615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876793465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876793465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784876801916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670348584651947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876818721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666654586791992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876835867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12688620388507843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876852337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12609434127807617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876852338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876852338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876879542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6624293923377991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876879542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876879542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784876897034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12025047093629837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876913979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976930499076843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876930363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178336888551712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876938623, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876938623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876964796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.669293224811554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876964796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876964797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784876972911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956385523080826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784876989628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664720624685287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877006159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12111291289329529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877022155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11840757727622986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877022155, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877022155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877050288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6982882618904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877050289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877050289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784877068097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975301802158356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877084754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084087193012238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877101339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664748936891556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877109578, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877109579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877135992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7255589365959167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877135993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877135993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784877144129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746588349342346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877161100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11718057841062546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877177650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223042011260986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877193671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557135730981827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877193764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877193764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877223195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7465112805366516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877223195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877223195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784877240034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420091986656189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877256565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379703879356384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877273172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681147873401642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877281394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877281395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877307199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7559648752212524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877307200, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877307200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784877315401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123735457658768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877332206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146212369203568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877348570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887308418750763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877365633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122060567140579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877365634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877365634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877393063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7643150687217712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877393064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877393064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784877409618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974457114934921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877426171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870744287967682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877442808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330638080835342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877450991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877450991, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877477055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670736908912659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877477055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877477056, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784877485127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240675508975983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877501792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127492785453796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877517897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474755614995956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877534237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821051150560379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877534238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877534238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877562426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7690516114234924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877562427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877562427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784877579097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119849756360054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877595643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849833488464355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877612233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750812292098999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877620222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877620223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877646541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701050639152527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877646542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877646542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784877654842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833859443664551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877671785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993903130292892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877688542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11104696989059448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877704495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385443270206451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877704496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877704496, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877731842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770939826965332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877731843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877731843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784877750094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355269700288773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877766888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799377202987671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877783918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464808881282806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877792360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877792361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877818883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713145017623901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877818883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877818883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784877827192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407429724931717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877844206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376967281103134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877860716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093425005674362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877876946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268207430839539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877876947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877876947, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877905569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717317342758179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877905569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877905569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784877922767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11706589162349701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877939704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11688587814569473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877956905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009058356285095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784877965467, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784877965467, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784877992045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720302939414978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784877992046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784877992046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784878000656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448546499013901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878017806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504314839839935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878034630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041316390037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878052703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364827513694763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878052704, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878052705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878079309, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721851468086243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878079309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878079310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784878096033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606614500284195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878113029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557246953248978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878129875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577499121427536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878138374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878138375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878165045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725644111633301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878165046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878165046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784878173455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355091094970703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878190754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108306348323822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878207555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703044384717941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878224906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225683987140656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878224907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878224908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878251697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728180885314941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878251697, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878251697, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784878268622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630032420158386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878285318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596829652786255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878302151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112152248620987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878310494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878310494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878337371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730467319488525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878337371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878337371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784878345916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198060423135757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878362873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090090334415436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878380025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094141006469727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878396445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341700911521912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878396446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878396446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878423985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730304002761841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878423985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878423985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784878441712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583388805389404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878458857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058521568775177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878475470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636318475008011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878484044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878484045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878510948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731840014457703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878510948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878510948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784878519515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066451221704483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878536244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935830533504486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878553322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763504356145859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878570277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384012013673782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878570278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878570278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878597138, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731637954711914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878597139, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878597139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784878613273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706353187561035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878630999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797331482172012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878647927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038292944431305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878656294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878656294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878682394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733307480812073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878682394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878682394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784878690618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085740178823471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878707511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070186197757721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878724196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115579903125763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878741125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780944675207138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878741126, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878741126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878767443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734217643737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878767443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878767444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784878784490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496828705072403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878800802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168004781007767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878818347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772636532783508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878827032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878827033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878853239, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773438572883606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878853240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878853240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784878861544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412139445543289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878878431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897157549858093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878895597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386825144290924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878912461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046722531318665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878912462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878912462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878938797, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734869718551636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878938798, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878938798, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784878955623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606662929058075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878972854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797479748725891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878989453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079455018043518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784878997547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784878997547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784879025021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735653519630432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784879025021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784879025021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784879034101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109782874584198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879051667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437786370515823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879069077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182302236557007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879086404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11264774203300476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879086404, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879086405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879113376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736684679985046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879113376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879113376, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784879130498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956050455570221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879147970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354139447212219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879165211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017386615276337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879173541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879173541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879200574, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737393379211426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879200574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879200574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784879208937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037389934062958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879227447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927202552556992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879244814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239590495824814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879262119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082178354263306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879262120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879262120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879289135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738435864448547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879289136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879289136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784879306413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991131514310837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879323986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235972493886948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879341257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169000193476677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879349745, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879349746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879377010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738432884216309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879377010, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879377010, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784879385236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463431268930435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879403372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744787007570267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879420241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094116628170013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879437241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686801373958588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879437242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879437242, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879464266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737768292427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879464266, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879464266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784879481209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467156022787094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879498137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596537590026855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879515339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502307116985321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879524039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879524040, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879550788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738203406333923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879550789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879550789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784879559090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833795368671417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879575375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309673845767975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879593553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104722760617733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879610567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725998133420944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879610568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879610568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879637527, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736971378326416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879637528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879637528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784879654400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080114752054214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879671453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385146737098694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879688087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329112410545349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879696470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879696470, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879723452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773865818977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879723453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879723453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784879732008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380733013153076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879749272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296894609928131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879765765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133434921503067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879783309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294903814792633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879783310, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879783310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879810767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739827036857605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879810767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879810768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784879827864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068851575255394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879844728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071871668100357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879861465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158921033143997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879869725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879869726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879896217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741191983222961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879896218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879896218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784879904441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118478536605835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879921202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10026373714208603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879938277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581312328577042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879955228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292205959558487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784879955228, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879955229, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879982248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740507125854492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879982248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879982248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784879998682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734647512435913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880017183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035865917801857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880034284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546319931745529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880042565, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880042566, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880069428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742621302604675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880069428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880069429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784880077908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328023880720139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880094910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801050066947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880111769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230499505996704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880129157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654216259717941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880129158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880129158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880156270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742007970809937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880156271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880156271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784880173660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019757091999054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880189957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636185854673386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880207003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920056700706482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880215747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880215748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880242507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742610573768616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880242508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880242508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784880251134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114501804113388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880267957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710757225751877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880284828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261788964271545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880301821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220258682966232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880301822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880301824, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880328160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742795944213867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880328160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880328161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784880344908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511620342731476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880362121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520771145820618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880379245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119186282157898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880387675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880387675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880414655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744277715682983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880414656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880414656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784880422726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702938586473465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880440267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252247542142868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880457928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887999832630157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880475066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273589730262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880475067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880475069, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880501376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746526002883911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880501376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880501377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784880518606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873370617628098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880536015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137348860502243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880553434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181853711605072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880562157, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880562157, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880588312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774747371673584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880588312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880588312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784880597060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212936043739319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880614064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174830049276352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880631524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071571335196495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784880649613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055875569581985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880649614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880649614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880676253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749077081680298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880676253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880676253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784880693308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375707387924194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880710585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850358009338379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880727883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111175537109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880736626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880736627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880764073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752801179885864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880764073, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880764074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784880772885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132730543613434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880790315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369984805583954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880807133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025257408618927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880825427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843939334154129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880825428, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880825428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880851671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753132581710815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880851671, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880851671, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784880868954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287339985370636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880886042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000678688287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880903277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017855629324913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880911927, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880911928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880938404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750768661499023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880938405, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880938405, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784880946901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997554287314415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880963844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061616450548172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880980965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210435837507248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880998101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562942922115326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880998102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784880998102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784881025340, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775119423866272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784881025341, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784881025341, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784881041383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428306460380554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881059165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997132211923599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881076185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058502048254013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881084571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881084572, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881111713, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775197446346283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881111713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881111713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784881119959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140497237443924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881136900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606551915407181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881153705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003783643245697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881170456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164888948202133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881170456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881170457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881197448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752478122711182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881197449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881197449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784881214537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020207479596138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881231924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320638120174408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881248263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149478167295456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881257509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881257509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881285632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756108045578003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881285632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881285633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784881294083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404146462678909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881311075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067739948630333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881327961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221640020608902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881345061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027649998664856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881345062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881345062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881371765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756236791610718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881371765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881371765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784881388955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522972792387009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881406053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139770805835724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881423240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233891010284424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881431951, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881431951, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881458109, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77572101354599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881458110, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881458110, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784881466706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159884184598923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881483125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948771238327026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881500330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742851346731186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881517475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794633626937866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881517475, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881517476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881544309, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759625911712646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881544310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881544310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784881561571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251496732234955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881578972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442511737346649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881596095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006203293800354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881604802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881604803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881631611, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764884233474731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881631612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881631612, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784881640392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487370193004608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881657733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407453775405884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881674909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689110308885574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881692266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087162122130394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881692267, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881692267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881719225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764785289764404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881719225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881719225, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784881735630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843661427497864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881754034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833916068077087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881771601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737835615873337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881780242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881780243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881806787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768930196762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881806788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881806788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784881815224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229072511196136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881832367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977190732955933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881849420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680942237377167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881866602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063469797372818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881866603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881866604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881893546, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772257328033447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881893546, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881893547, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784881911064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642879456281662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881928347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10493206977844238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881945743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032192409038544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881953922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881953923, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881981358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770702242851257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881981358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881981358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784881989843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796862095594406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882008314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963757336139679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882026242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005446314811707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882043989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169590055942535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882043989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882043990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882070839, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779525518417358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882070839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882070840, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784882088406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403624713420868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882105890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278814822435379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882123327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101000010967255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882132178, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882132178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882158515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781747579574585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882158515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882158515, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784882167103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081387847661972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882184580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237704753875732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882201318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379911750555038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882219671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854514479637146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.2131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882219672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882219672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882246627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781659364700317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882246627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882246627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784882264368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11686407774686813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882282133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318659782409668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882299941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312167346477509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882308794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882308795, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882335210, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778802752494812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882335210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882335210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784882343890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143522635102272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882361530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457681655883789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882379001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425282061100006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882395833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731891334056854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882395834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882395834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882423104, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779157280921936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882423105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882423105, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784882441716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908744484186172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882459091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146826297044754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.2431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882476395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342257261276245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882485498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882485499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882511331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786187529563904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882511331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882511331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784882519968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992269963026047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882537344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193664371967316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882555039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719948261976242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882572658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701747983694077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882572658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882572659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882600028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791222333908081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882600028, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882600028, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784882616959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362102627754211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.2631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882635741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865205526351929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882653276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759884864091873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882661980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882661981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882689117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789472341537476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882689118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882689118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784882697482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543449223041534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882715037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442384332418442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882732521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383749008178711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882749898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873693227767944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.2831666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882749899, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882749900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882776612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791144847869873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882776613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882776613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784882794410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039336025714874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882812013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517846792936325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882828890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753601789474487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882838075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882838076, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882866174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797883152961731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882866174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882866174, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784882874794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642007738351822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882892237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593937337398529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882909454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864928364753723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.3031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882926957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070941612124443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882926958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882926958, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882953712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803550958633423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882953713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882953713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784882970749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669394582509995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784882987830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427002608776093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883005299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028098538517952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883014017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883014017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883040600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797175049781799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883040600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883040601, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784883049410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121314764022827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3198333333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883066244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581167042255402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.3231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883084106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480302572250366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883101606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050860807299614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883101606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883101607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883128369, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802764177322388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883128369, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883128370, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784883145872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296987742185593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262144000, "lr": 1.3331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883163326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099201649427414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262799360, "lr": 1.3365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883180787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465098917484283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 263454720, "lr": 1.3398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883189491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883189491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883216007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800474762916565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883216007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883216007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784883224735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853356122970581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264110080, "lr": 1.3431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883242503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728973150253296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264765440, "lr": 1.3465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883260364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451675951480865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 265420800, "lr": 1.3498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883277710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300679504871368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266076160, "lr": 1.3531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883277711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883277711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883305375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808418869972229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883305376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883305376, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784883322181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028703898191452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266731520, "lr": 1.3565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883341087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622532665729523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 267386880, "lr": 1.359833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883358538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077176034450531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268042240, "lr": 1.3631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883367386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883367386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883394587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802652716636658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883394587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883394587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784883403171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11054477095603943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268697600, "lr": 1.3665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883420674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468289256095886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 269352960, "lr": 1.3698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883438125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032881811261177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270008320, "lr": 1.3731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883455616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878737270832062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270663680, "lr": 1.3765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883455617, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883455617, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883482320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.781043529510498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883482321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883482321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784883500211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104065477848053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271319040, "lr": 1.379833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883517993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092062294483185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271974400, "lr": 1.3831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883535176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827282071113586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 272629760, "lr": 1.3865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883543708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883543708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883571304, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808769941329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883571305, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883571305, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784883579900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149346083402634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273285120, "lr": 1.3898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883598359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109781414270401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273940480, "lr": 1.3931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883615946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698111146688461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 274595840, "lr": 1.3965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883633435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152462363243103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275251200, "lr": 1.3998333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883633435, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883633435, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883659940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780853807926178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883659940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883659940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784883677707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112719029188156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275906560, "lr": 1.4031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883695256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986240208148956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 276561920, "lr": 1.4065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883713214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704541742801666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277217280, "lr": 1.4098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883722075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883722075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883748596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.781636655330658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883748597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883748597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784883757549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222244054079056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277872640, "lr": 1.4131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883774716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855716466903687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 278528000, "lr": 1.4165000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883793132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667680740356445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279183360, "lr": 1.4198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883811148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864514112472534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279838720, "lr": 1.4231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883811149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883811149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883837931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816686630249023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883837931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883837931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784883855336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099278181791306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 280494080, "lr": 1.4265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883873087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088529229164124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281149440, "lr": 1.4298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883890778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520281434059143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281804800, "lr": 1.4331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883899551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883899552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883925543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813778519630432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883925543, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883925543, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784883934592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578251630067825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 282460160, "lr": 1.4365000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883952469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305507272481918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283115520, "lr": 1.4398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883969955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061988770961761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283770880, "lr": 1.4431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883987160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111774891614914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 284426240, "lr": 1.4464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784883987161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784883987162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784884016680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816402912139893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784884016680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784884016680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784884034604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766953229904175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285081600, "lr": 1.4498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884052226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220499128103256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285736960, "lr": 1.4531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884069952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374041438102722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 286392320, "lr": 1.4564999999999997e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884078870, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884078871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884105647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814568281173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884105648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884105648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784884114406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381067335605621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287047680, "lr": 1.4598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884132523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051641404628754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287703040, "lr": 1.4631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884150494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242207884788513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 288358400, "lr": 1.4664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884167635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255357414484024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289013760, "lr": 1.4698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884167636, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884167636, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884194339, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820760011672974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884194339, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884194340, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784884211647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707518458366394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289669120, "lr": 1.4731666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884228841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487651824951172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290324480, "lr": 1.4764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884247214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053040832281113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290979840, "lr": 1.4798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884255942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884255943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884282414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816370725631714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884282415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884282415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784884291228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478147119283676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 291635200, "lr": 1.4831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884308628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847786068916321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292290560, "lr": 1.4864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884326034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464003682136536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292945920, "lr": 1.4898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884343433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052820086479187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 293601280, "lr": 1.4931666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884343434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884343434, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884369928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815587520599365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884369929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884369929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784884387040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738224536180496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294256640, "lr": 1.4964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884404599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345128178596497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294912000, "lr": 1.4998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884422398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768543183803558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 295567360, "lr": 1.5031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884431247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884431248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884457971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820292711257935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884457971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884457971, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784884466559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391265153884888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296222720, "lr": 1.5064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884483383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419527441263199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296878080, "lr": 1.5098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884502011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822319984436035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 297533440, "lr": 1.5131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884519649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928736627101898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298188800, "lr": 1.5164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884519649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884519650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884546145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818660140037537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884546145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884546145, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784884563727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512958467006683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298844160, "lr": 1.5198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884581296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027807667851448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 299499520, "lr": 1.5231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884598711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381969064474106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300154880, "lr": 1.5265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884607595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884607596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884634310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819225192070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884634310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884634311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784884643122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020181328058243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300810240, "lr": 1.5298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884660493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09947437793016434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 301465600, "lr": 1.5331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884678021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10074611008167267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302120960, "lr": 1.5364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884695773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067749634385109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302776320, "lr": 1.5398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884695774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884695774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884722632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820176482200623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884722632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884722632, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784884739338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092187762260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 303431680, "lr": 1.5431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884757239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603762418031693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304087040, "lr": 1.5465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884774867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271081328392029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304742400, "lr": 1.5498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884783569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884783569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884810469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823720574378967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884810470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884810470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784884819247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143186151981354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 305397760, "lr": 1.5531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884836571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721389204263687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306053120, "lr": 1.5564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884853968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487998276948929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306708480, "lr": 1.5598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884871175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511496663093567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 307363840, "lr": 1.5631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884871175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884871176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884897624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7821642160415649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884897624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884897625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784884915099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639236122369766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308019200, "lr": 1.5665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884932587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981251671910286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308674560, "lr": 1.5698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884950038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373543947935104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309329920, "lr": 1.5731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784884958717, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884958717, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884984864, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820995450019836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884984865, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884984865, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784884993538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774700343608856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309985280, "lr": 1.5764999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885009912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133875161409378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 310640640, "lr": 1.5798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885028277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858563333749771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311296000, "lr": 1.5831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885046143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723941773176193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311951360, "lr": 1.5865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885046144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885046144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885072804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826921939849854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885072804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885072805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784885090404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498051345348358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 312606720, "lr": 1.5898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885107960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607510805130005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313262080, "lr": 1.5931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885125583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513058304786682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313917440, "lr": 1.5964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885134245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885134246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885161102, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826122641563416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885161102, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885161102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784885169844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795781761407852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 314572800, "lr": 1.5998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885187271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040109246969223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315228160, "lr": 1.6031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885205106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326971113681793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315883520, "lr": 1.6065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885222673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239951312541962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 316538880, "lr": 1.6098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885222674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885222674, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885249367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828235626220703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885249367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885249367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784885267230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710550844669342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317194240, "lr": 1.6131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885284127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854456573724747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317849600, "lr": 1.6165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885302306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203959584236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 318504960, "lr": 1.6198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885311373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885311373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885337559, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829596996307373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885337560, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885337560, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784885346477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328011751174927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319160320, "lr": 1.6231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885363991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393938422203064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319815680, "lr": 1.6265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885381495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281699687242508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 320471040, "lr": 1.6298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885399083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704928636550903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321126400, "lr": 1.6331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885399084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885399085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885425520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825653553009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885425520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885425520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784885442837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030997335910797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321781760, "lr": 1.6365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885460506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835695266723633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 322437120, "lr": 1.6398333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885478402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175896972417831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323092480, "lr": 1.6431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885487364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885487365, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885514116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833569049835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885514116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885514116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784885523111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886958241462708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323747840, "lr": 1.6465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885540817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411982238292694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 324403200, "lr": 1.6498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885557825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521664470434189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325058560, "lr": 1.6531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885576898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272744834423065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325713920, "lr": 1.6565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885576898, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885576899, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885603068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783111572265625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885603068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885603068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784885621014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938086360692978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 326369280, "lr": 1.6598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885638659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157510429620743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327024640, "lr": 1.6631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885656738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702474415302277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680000, "lr": 1.6665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885665628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885665628, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885692014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829775810241699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885692014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885692015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784885700791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359578371047974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328335360, "lr": 1.6698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885718620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077975332736969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328990720, "lr": 1.6731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885736710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246925592422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 329646080, "lr": 1.6765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885754608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854213684797287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330301440, "lr": 1.6798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885754609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885754609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885781449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834774255752563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885781449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885781449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784885798436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477530747652054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330956800, "lr": 1.6831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885817143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142729222774506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 331612160, "lr": 1.6865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885834951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296297818422318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332267520, "lr": 1.6898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885843829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885843830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885870843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832448482513428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885870844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885870844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784885879641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660481452941895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332922880, "lr": 1.6931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885896974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434991866350174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 333578240, "lr": 1.6964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885914550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072196289896965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334233600, "lr": 1.6998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885931976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094915330410004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334888960, "lr": 1.7031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885931977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885931977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885958407, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836584448814392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885958407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885958407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784885975842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468810051679611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 335544320, "lr": 1.7064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784885993547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637696087360382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336199680, "lr": 1.7098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886011126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800019651651382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336855040, "lr": 1.7131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886020016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886020017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886046443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7835452556610107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886046444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886046444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784886055385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968975186347961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 337510400, "lr": 1.7164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886072222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133862495422363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338165760, "lr": 1.7198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886090492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700932145118713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338821120, "lr": 1.7231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886108358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191167891025543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 339476480, "lr": 1.7265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886108358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886108359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886135415, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831670641899109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886135415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886135416, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784886153114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120445489883423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340131840, "lr": 1.7298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886170765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352165997028351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340787200, "lr": 1.7331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886188341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605931282043457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 341442560, "lr": 1.7364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886197081, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886197082, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886224304, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784066915512085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886224305, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886224305, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784886233076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288579761981964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342097920, "lr": 1.7398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886250530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118129104375839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342753280, "lr": 1.7431666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886268232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591751337051392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 343408640, "lr": 1.7465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886285926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832994431257248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344064000, "lr": 1.7498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886285926, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886285927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886312696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839410305023193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886312696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886312696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784886330436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591191053390503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344719360, "lr": 1.7531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886347347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773947834968567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 345374720, "lr": 1.7564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886364252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953959286212921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346030080, "lr": 1.7598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886374233, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886374233, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886401327, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836059927940369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886401328, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886401328, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784886410418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040981262922287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346685440, "lr": 1.7631666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886428030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027936339378357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347340800, "lr": 1.7665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886445684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634923726320267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347996160, "lr": 1.7698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886463053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10113748908042908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 348651520, "lr": 1.7731666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886463124, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886463124, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886489762, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840479612350464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886489762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886489762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784886507311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862861573696136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349306880, "lr": 1.7764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886524936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735245794057846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349962240, "lr": 1.7798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886542495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039494127035141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 350617600, "lr": 1.7831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886551317, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886551318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886578175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839494943618774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886578175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886578176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784886586951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457011312246323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351272960, "lr": 1.7865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886604598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028287768363953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351928320, "lr": 1.7898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886622155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993876516819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 352583680, "lr": 1.7931666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886638738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535379499197006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353239040, "lr": 1.7964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886638801, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886638801, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886665627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7842042446136475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886665628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886665628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784886684027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617722570896149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353894400, "lr": 1.7998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886701677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053471639752388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 354549760, "lr": 1.8031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886718778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617402195930481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355205120, "lr": 1.8065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784886727276, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784886727276, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed451547632.log b/recommendation_v4/rcp_logs/gbs_32768/seed451547632.log new file mode 100644 index 000000000..373d34d52 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed451547632.log @@ -0,0 +1,888 @@ +:::MLLOG {"namespace": "", "time_ms": 1784881834498, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784881834498, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784881863128, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "seed", "value": 451547632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784881863130, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784881863131, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784881863134, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784881863134, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784882734844, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784882734845, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784882735147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784882909560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384854018688202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882925031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846956193447113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882939455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855421543121338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784882946729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882946729, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882970865, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4985979497432709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882970866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882970866, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784882978098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844791054725647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784882992444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849245011806488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883007409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838844001293182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883022132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844826817512512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883022133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883022133, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883045050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49874135851860046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883045051, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883045051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784883059813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815146684646606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883074516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382545530796051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883089513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810575008392334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883096738, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883096739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883120184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989156424999237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883120185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883120185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784883127903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820409774780273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883142667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817428052425385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883157868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825726509094238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883172748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780304789543152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883172749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883172749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883196001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4991208612918854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883196001, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883196001, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784883210588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811808824539185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883225501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751903176307678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883240556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378563493490219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883247859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883247860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883270871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4993593990802765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883270871, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883270871, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784883277988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710342347621918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883293116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719689846038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883307808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689610362052917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883322862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693523406982422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883322863, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883322863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883346966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49963119626045227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883346967, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883346967, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784883361496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371329128742218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883376238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700729608535767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883391913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637614250183105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883399468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883399469, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883423179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49990007281303406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883423180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883423180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784883430807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718068599700928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883446747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614732027053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883461787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649728894233704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883477869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685129582881927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883477869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883477870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883501696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002606511116028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883501696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883501696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784883516819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617272675037384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883532605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482511043548584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784883547509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657855987548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883554968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883554968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883579237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006150007247925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883579237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883579237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784883586648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309909403324127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883602182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345905065536499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883616946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502156734466553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883631777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517574965953827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883631844, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883631844, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883655846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009852051734924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883655846, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883655846, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784883670722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379567861557007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883685576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462378084659576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883700356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514432311058044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883708501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883708502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883731935, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014122724533081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883731936, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883731936, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784883739247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569791615009308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883753978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490717113018036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883769108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321820318698883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883784736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344946324825287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883784737, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883784737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883807330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019556283950806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883807330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883807331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784883822544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336936503648758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883837980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520751893520355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883853923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350414752960205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883861632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883861632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883885091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028138160705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883885091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883885092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784883892541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632610440254211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883907758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335156410932541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883923628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343071460723877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883938712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321495920419693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883938779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883938779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883962088, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037333965301514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883962089, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883962089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784883977024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329214870929718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784883992733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13123804330825806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884007622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311107456684113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884015204, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884015205, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884038387, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047145485877991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884038387, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884038387, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784884045919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382797509431839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884061046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311229020357132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884076784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994074821472168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884091793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037902116775513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884091793, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884091794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884114997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058517456054688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884114998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884114998, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784884129955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327379047870636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884145903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976643443107605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884162805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306706815958023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884170614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884170614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884193699, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070832371711731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884193699, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884193699, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784884201379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315518170595169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884216401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520045578479767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884231826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331876516342163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884247538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356295794248581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884247538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884247539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884270504, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5084412693977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884270505, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884270505, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784884285765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352328509092331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884300947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262133300304413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884316227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332818865776062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884323794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884323794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884347227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5098029375076294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884347228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884347228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784884355710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637012243270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884371371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353350430727005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884386783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620281219482422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884402495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649961352348328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884402496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884402496, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884426204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112141370773315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884426205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884426205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784884442617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482657074928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884458324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308828443288803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884473875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632959127426147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884481785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884481786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884505157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125888586044312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884505158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884505158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784884512946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602668046951294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884528761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257287442684174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884543973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326364427804947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884559147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082876801490784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884559148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884559148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884582002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138918161392212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884582002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884582002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784884597240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101013004779816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884613295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819396495819092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884628827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295868456363678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884636510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884636510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884659320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.515356719493866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884659321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884659321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784884666980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929759919643402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884682543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475418090820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884698010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466393947601318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884714230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046078383922577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884714231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884714231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884737066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5171405076980591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884737067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884737067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784884752191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12818178534507751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884767420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13180766999721527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884782948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555274903774261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884790733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884790734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884814622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5190075039863586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884814622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884814623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784884822256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662786781787872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884838150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355585932731628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884853382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209594786167145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884868619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100934028625488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884868620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884868620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884891781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5212580561637878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884891782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884891782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784884907348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12957559525966644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884922866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253560662269592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884938873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546618819236755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884946661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884946662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884969709, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238773226737976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884969710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884969710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784884977300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198041915893555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784884992992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335175782442093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885008634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497677445411682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885024366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13461826741695404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885024367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885024367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885048206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5268421769142151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885048207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885048207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784885063979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598506152629852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885080349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339123398065567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885096264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317953050136566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885104041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885104042, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885127513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.530098021030426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885127513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885127514, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784885135589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403830647468567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885151709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526974618434906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885168168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451947271823883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885184326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538914918899536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885184327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885184327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885207460, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.533525288105011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885207461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885207461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784885223218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520905375480652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885239092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329973816871643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885254925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231799006462097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885263392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885263393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885287163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5373422503471375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885287163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885287163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784885295154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325460523366928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885310744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097858428955078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885326201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595107197761536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885341997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324041187763214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885341998, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885341998, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885366253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5419042706489563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885366253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885366253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784885382041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489094376564026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885398374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302386224269867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885414075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319689303636551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885421856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885421856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885444524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5480696558952332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885444525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885444525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784885452183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234473764896393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885467953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308414489030838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885483809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315989077091217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885500086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326913982629776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885500087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885500087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885523835, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5564262270927429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885523835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885523835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784885539368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321081519126892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885554760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283462643623352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885569898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341208815574646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885577749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885577750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885600496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5660231709480286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885600497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885600497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784885608409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487836718559265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885624003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269482672214508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885640179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718407809734344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885656042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133131742477417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885656043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885656043, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885679400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5772469639778137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885679401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885679401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784885694855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12947459518909454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885710388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289048194885254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885726312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820348143577576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885733989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885733989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885756858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5883010029792786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885756859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885756859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784885764825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033714890480042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885781142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279478669166565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885796869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037243485450745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885812523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287526786327362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885812523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885812524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885834858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6000393033027649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885834858, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885834859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784885850314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12716662883758545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885866203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13007861375808716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885881961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331890106201172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885889838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885889838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885912824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6104844212532043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885912825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885912825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784885921287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178789615631104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885937657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12730303406715393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885953947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318204253911972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885970062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271214783191681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784885970063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885970063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885992626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6203960180282593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885992627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784885992627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784886009018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12962038815021515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886025196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059818744659424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886041915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12838490307331085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886050296, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886050297, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886073154, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6303582787513733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886073155, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886073155, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784886081319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769338488578796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886097518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12754881381988525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886113863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285361498594284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886130253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275898516178131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886130253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886130253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886154088, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6381576657295227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886154088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886154089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784886170663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12207625806331635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886186669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12101994454860687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886202405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956822872161865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886210363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886210364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886233305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6430808305740356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886233306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886233306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784886241043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12113345414400101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886257089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181231290102005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886273045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12238481640815735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886288634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998529732227325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886288635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886288635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886311878, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6583248376846313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886311879, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886311879, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784886328600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12113495171070099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886344432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231429874897003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886360233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11802317947149277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886368019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886368019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886390891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.675504744052887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886390892, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886390892, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784886398691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11864148080348969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886414693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11875653266906738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886430659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339452862739563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886446221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708558350801468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886446293, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886446294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886471170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6990647912025452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886471171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886471171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784886487267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11571413278579712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886503029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592039465904236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886519131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897403210401535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886527132, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886527132, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886550440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7209038734436035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886550441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886550441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784886558413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281952261924744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886574736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141560897231102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886590920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247013509273529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886607321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404088139533997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886607322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886607322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886631311, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7425894737243652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886631311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886631312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784886647047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283911764621735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886662751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120222806930542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886678620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622168123722076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886686426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886686427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886710149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7532435655593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886710150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886710150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784886717952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578270256519318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886734179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11315152049064636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886750119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073826402425766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886766377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156688630580902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886766377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886766378, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886790796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7575405240058899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886790796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886790796, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784886807047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411601305007935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886822874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149115860462189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886838665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993040353059769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886846367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886846368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886870596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7607637643814087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886870597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886870597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784886878474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981893539428711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886894482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144974827766418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886910679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196261644363403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886926255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590637475252151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886926255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886926256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886950316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619407176971436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886950317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886950317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784886967521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11511506140232086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886983698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11873918771743774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784886999945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11585532873868942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887007996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887007997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887031732, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7631088495254517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887031733, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887031733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784887039694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489902436733246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887056134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458874493837357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887072292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118176057934761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887088309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288157105445862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887088310, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887088310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887112108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7641599774360657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887112108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887112109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784887128477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750884354114532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887144808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11722282320261002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887161268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035475134849548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887169483, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887169483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887193455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652466297149658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887193456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887193456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784887201699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494617909193039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887218288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494610458612442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887234571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056286096572876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887251414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039777398109436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887251415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887251415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887274915, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7667369842529297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887274916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887274916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784887290484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612562298774719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887306484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539884120225906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887322373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544685274362564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887330303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887330303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887354123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677466869354248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887354124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887354124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784887362051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378299653530121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887378370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120280623435974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887394510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708394646644592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887411145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254532635211945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887411145, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887411146, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887434178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768333911895752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887434178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887434178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784887450158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654276609420776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887465764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604039579629898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887481859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149441659450531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887489724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887489724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887512933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691887021064758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887512934, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887512934, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784887520958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021139919757843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887537110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121025145053864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887553491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119026690721512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887569335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373054444789886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887569336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887569336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887593153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699817419052124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887593153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887593154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784887609564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059487983584404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887625742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061072126030922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887641354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064687967300415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887649566, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887649566, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887673086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705470323562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887673086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887673087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784887681190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719600319862366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887697101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936170279979706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887713366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750710964202881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887729725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406427085399628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887729725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887729726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887753955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710296511650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887753955, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887753956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784887769779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745806246995926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887786522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785879194736481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887802656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401144623756409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887810593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887810594, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887833721, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771191418170929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887833722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887833722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784887841606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878447443246841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887857781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713283717632294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887873768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170347779989243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887889962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786720365285873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887889963, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887889963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887913108, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715926766395569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887913109, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887913110, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784887929567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502666234970093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887945444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169744491577148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887962256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761388391256332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784887970654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887970654, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887993837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719863653182983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887993837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784887993838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784888001978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453122854232788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888018081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09901329129934311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888034469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382275819778442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888050693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072532832622528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888050694, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888050694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888074574, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721026539802551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888074575, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888074575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784888090797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622035712003708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888107469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846894979476929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888123761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799203813076019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888131729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888131730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888156079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723450660705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888156079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888156079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784888164836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995133221149445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888181393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437161266803741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888197839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200709640979767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888214288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302826553583145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888214288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888214288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888238225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725673913955688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888238225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888238226, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784888254331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963433980941772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888270975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375337094068527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888287527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048053205013275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888295697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888295698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888320328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726718187332153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888320328, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888320328, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784888328408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038023978471756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888345538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091945469379425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888362118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258844286203384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888378688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076980829238892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888378689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888378689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888402544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729315161705017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888402544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888402545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888419161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992637276649475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888436119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126999482512474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888452909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698555946350098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888461303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888461303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888485153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730462551116943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888485154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888485154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784888493193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476169526576996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888510201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750459134578705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888526601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135578364133835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888542828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673581808805466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888542829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888542829, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888567054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730514407157898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888567055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888567055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784888583219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490082204341888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888599159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618437826633453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888615634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515834391117096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888623920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888623920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888648430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730526924133301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888648430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888648431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784888656584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851341485977173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888672485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297362506389618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888689460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045868843793869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888705716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723283886909485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888705716, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888705717, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888729235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732975482940674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888729236, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888729236, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784888745090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004183366894722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888761306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356836766004562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888777437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315417498350143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888785451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888785451, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888808667, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773424506187439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888808668, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888808668, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784888816777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038614884018898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888833586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288657248020172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888849803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389464139938354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888866382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312244296073914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888866383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888866383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888890352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736362814903259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888890352, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888890353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784888906644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704539716243744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888922761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074783131480217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888938917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137490928173065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888946991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888946991, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888971261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738037705421448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888971262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888971262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784888979222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129030048847198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888995433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046500712633133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889011864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608832538127899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889028317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321349650621414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889028318, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889028318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889052643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773750901222229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889052644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889052644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784889068461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073622927069664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889085447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357165336608887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889101709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582283139228821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889109672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889109673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889133172, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740002274513245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889133173, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889133173, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784889141307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348125547170639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889157679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815732926130295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889173981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257329791784286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889190585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650776326656342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889190586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889190586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889219317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740384340286255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889222279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889222279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784889238926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207125544548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889259540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636487603187561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889285781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962066054344177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889316805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889321105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889344811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742623090744019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889344811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889344811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784889366940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177186667919159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889407893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716742277145386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889460758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285426676273346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889506852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232706367969513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889506853, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889506854, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889530822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743484377861023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889530823, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889530823, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784889579256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487593710422516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889653645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540784895420074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889716789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143623292446136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889733778, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889733779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889757962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746222615242004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889757962, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889757962, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784889769404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702411830425262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889786514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245780438184738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889803576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873749852180481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889820113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266264319419861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889820114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889820114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889843417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745409607887268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889843418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889843418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784889860008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864926874637604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889876713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401481181383133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889893461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180715262889862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889901906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889901907, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889925497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747890949249268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889925497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889925498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784889933972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123509630560875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889950870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189191043376923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889968282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708567500114441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889985750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070600152015686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784889985751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784889985752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784890009159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749075889587402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784890009159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784890009159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784890025755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395107209682465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890042528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857507586479187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890059403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130531132221222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890067993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890067994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890091695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749379873275757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890091696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890091696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784890100223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114620715379715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890117327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039004921913147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890133899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987570881843567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890151062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803408175706863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890151063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890151063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890174737, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753698825836182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890174738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890174738, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784890191118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034744456410408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890207407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988760203123093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890223846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200152546167374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890232062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890232062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890255799, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749087810516357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890255799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890255800, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784890263881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954259544610977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890280053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10059131681919098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890296346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226759314537048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890313015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054251492023468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890313016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890313016, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890337060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750999331474304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890337060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890337060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784890353008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401719063520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890370209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979333519935608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890386951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600004345178604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890395160, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890395161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890418781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753608226776123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890418782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890418782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784890426905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143493115901947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890443720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558691620826721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890460245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003618910908699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890476674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171882808208466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890476675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890476675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890499908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755463123321533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890499909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890499909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784890516481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192404687404633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890534663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308690369129181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890550947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167673975229263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890559850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890559850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890583714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754968404769897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890583714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890583714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784890591801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389401763677597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890608161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697809606790543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890625619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247914493083954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890642029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033334791660309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890642029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890642030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890666028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756677865982056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890666029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890666029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784890682270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560767352581024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890698498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177922248840332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890715040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101817287504673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890723352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890723353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890746115, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755540013313293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890746116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890746116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784890754422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11161994189023972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890770586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933602601289749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890787040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765137523412704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890803683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814084857702255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890803684, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890803684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890826253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775669515132904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890826253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890826253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784890842571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288897901773453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890858920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045256182551384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890875326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063591599464417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890883615, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890883616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890906861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762965559959412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890906862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890906862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784890915236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053745299577713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890932344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407590121030807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890948848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699744522571564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890965676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887517035007477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784890965676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890965676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890989136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762446999549866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890989137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784890989137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784891005349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853388905525208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891022349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867057740688324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891039103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745660960674286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891047307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891047307, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891070253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767040133476257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891070254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891070254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784891078176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238779127597809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891094455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983629524707794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891110842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680873692035675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891130844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654565691947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891130847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891130848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891154789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767565250396729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891154789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891154789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784891171288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106717050075531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891188931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053004115819931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891205529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110427625477314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891213551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891213551, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891237856, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766452431678772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891237856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891237856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784891246078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108339823782444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891263451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983248054981232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891280436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051743477582932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891297384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183653026819229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891297384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891297385, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891320620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771741151809692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891320621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891320621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784891337659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423642188310623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784891354502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336721479892731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed457724564.log b/recommendation_v4/rcp_logs/gbs_32768/seed457724564.log new file mode 100644 index 000000000..171b321dd --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed457724564.log @@ -0,0 +1,687 @@ +:::MLLOG {"namespace": "", "time_ms": 1784956263534, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784956263534, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784956286929, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "seed", "value": 457724564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784956286935, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784956286973, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784956286974, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784957130556, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784957130557, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784957130877, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784957340100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891899585723877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784957355386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890013098716736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784957369477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882321119308472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784957376583, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957376584, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957400067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013392567634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957400068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957400068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784957407107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883662223815918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957421199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874632120132446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957435788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877946138381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957450181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863061368465424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957450182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957450182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957472940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014820694923401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957472940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957472940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784957487230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863490521907806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957501517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863971829414368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957516139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858960568904877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957523258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957523259, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957547206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016613602638245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957547206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957547206, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784957554791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859862089157104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957568964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138441264629364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957584002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384638100862503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957598577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382390856742859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957598577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957598578, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957624212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018575191497803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957624212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957624212, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784957638472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382429450750351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957653163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790687918663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957667694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802769780158997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957674881, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957674881, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957700453, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020803809165955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957700454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957700454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784957707385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750620186328888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957722303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375921666622162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957736481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723015785217285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957751114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372499316930771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957751115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957751115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957776542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023305416107178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957776543, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957776543, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784957790497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742339611053467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957804974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373400241136551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957820114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367240697145462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957827558, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957827559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957851164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025795102119446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957851165, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957851166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957858688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742467761039734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957874258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364569067955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957889171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678450882434845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957904793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372275948524475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957904794, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957904794, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957930703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029360055923462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957930703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957930704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784957945547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647456467151642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957960854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352510154247284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784957975517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366303563117981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957982879, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784957982879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784958005791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032640099525452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784958005792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784958005792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784958013195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335064321756363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958028275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492649793624878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958042852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522279262542725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958057434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540643453598022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958057506, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958057506, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958083257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036149024963379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958083258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958083258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784958097924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409343361854553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958112835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473640382289886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958127568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528616726398468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958135504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958135505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958161105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5040175914764404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958161106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958161106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784958168326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579820096492767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958183346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493755459785461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958198393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222503662109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958213988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346449851989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958213988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958213989, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958239771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045293569564819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958239772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958239772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784958255058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389813899993896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958270451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518625497817993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958286148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498398661613464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958293894, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958293894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958319126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053638815879822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958319127, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958319127, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784958326632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637349009513855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958341886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354091346263885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958357778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342584490776062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958372948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321282535791397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958373013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958373014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958397629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.506300687789917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958397630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958397630, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784958412607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280968368053436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958428252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125455379486084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958443345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320094347000122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958450802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958450803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958476822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072986483573914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958476823, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958476823, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784958484218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843291997909546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958499438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110795617103577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958514931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12990978360176086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958530127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304004043340683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958530128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958530128, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958555838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5084630846977234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958555839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958555839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784958570734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278508186340332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958585800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975621223449707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958601202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305902600288391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958608779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958608779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958633765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5098000168800354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958633766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958633766, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784958641208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169744610786438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958656079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351403146982193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958671275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330956518650055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958686909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567262887954712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958686909, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958686910, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958711653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112242698669434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958711653, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958711653, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784958726980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522721827030182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958742238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251903653144836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958757632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330910623073578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958765208, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958765209, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958788110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5126243829727173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958788110, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958788110, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784958796641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635340332984924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958812191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352698802947998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958827862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620683550834656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958843547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650059700012207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958843548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958843548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958867250, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5140380263328552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958867250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958867251, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784958883485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482026755809784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958899531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083364069461823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958915082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629522919654846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958923048, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958923048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958947153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5153489708900452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958947154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958947154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784958954841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592126965522766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958970487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248437643051147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784958985948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326395571231842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959001311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089875876903534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959001311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959001311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959026583, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5166118144989014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959026584, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959026584, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784959041744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102656602859497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959057735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382453441619873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959073239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295485496520996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959080887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959080888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959106189, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5181728005409241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959106190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959106190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784959113698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924614548683167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959129116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474369049072266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959144537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345541775226593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959160561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027578592300415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959160562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959160562, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959186310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5200087428092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959186310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959186310, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784959201878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809327244758606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959217062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179782032966614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959232596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561636209487915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959240321, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959240322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959264294, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5218887329101562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959264295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959264295, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784959272092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365503966808319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959288051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365384936332703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959303731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13213074207305908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959319191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13095998764038086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959319192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959319192, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959345176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5240512490272522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959345176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959345177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784959360814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961772084236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959376260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325347125530243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959392286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534465432167053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959400226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959400227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959426065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5267432332038879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959426066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959426066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784959433865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197660446166992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959449524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339293003082275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959465242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349182426929474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959480841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346169263124466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959480842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959480842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959505435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5296194553375244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959505435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959505435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784959520916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604791462421417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959537480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395604491233826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959553582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331167221069336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959561413, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959561413, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959586751, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5330262184143066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959586752, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959586752, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784959595000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403738260269165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959610872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527831435203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959627128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448156416416168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959643823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522355258464813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959643824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959643824, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959669879, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5363401174545288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959669880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959669880, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784959685838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523399829864502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959701824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320010900497437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959717614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237221539020538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959726077, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959726077, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959752130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5401883125305176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959752131, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959752131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784959759923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325775682926178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959775729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084883987903595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959791032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359279304742813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959806826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239046931266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959806827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959806827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959830566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5445753931999207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959830566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959830566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784959846311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478881120681763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959862773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036870956420898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959878874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321103572845459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959886799, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959886799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959912655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5505914688110352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959912656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959912656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784959920487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245789706707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959936347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093248009681702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959952297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320422172546387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959968230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281871378421783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784959968231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784959968232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784959994741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.559337317943573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784959994742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784959994742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784960010214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320902705192566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960025793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128410205245018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960041187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343182623386383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960049096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960049097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960074853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5687724947929382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960074854, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960074855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784960082657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510572910308838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960098277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704147398471832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960114323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734690308570862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960130503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327521085739136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960130504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960130504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960156613, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5813794136047363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960156614, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960156614, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784960172290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297808438539505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960188336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890803813934326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960204653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286291778087616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960212372, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960212373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960235674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5941627621650696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960235675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960235675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784960243585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046184182167053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960259897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824422121047974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960275578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058169186115265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960291086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290348470211029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960291087, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960291087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960317364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6092924475669861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960317365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960317365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784960333073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745904922485352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960348954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037744164466858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960364618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381421566009521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960372359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960372359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960396034, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6240498423576355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960396035, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960396035, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784960404510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321459859609604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960420896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275506317615509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960437253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13186490535736084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960453405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12767748534679413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960453405, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960453406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960479607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6393148899078369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960479608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960479608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784960495863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000720739364624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960511946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117018342018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960528457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290844976902008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960536603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960536604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960562276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6555496454238892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960562277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960562277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784960570451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281774640083313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960586309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809687852859497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960602502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934815883636475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960618531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12835942208766937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960618531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960618532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960641658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6733044981956482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960641659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960641659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784960657904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12256608158349991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960673661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12180937081575394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960689164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12054748088121414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960697057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960697058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960722730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6898194551467896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960722730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960722730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784960730494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12175504863262177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960746368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11903546750545502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960762244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296450883150101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960777866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063363939523697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960777866, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960777867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960801453, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7108113765716553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960801454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960801454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784960817928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12159843742847443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960833867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331857740879059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960849386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11875748634338379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960857157, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960857157, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960882123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7264090180397034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960882124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960882124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784960889736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11893691122531891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960905681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922790110111237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960921398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136632114648819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960936916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717651039361954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960936982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960936982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960962666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7344011664390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960962667, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960962667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784960978463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11555855721235275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784960994236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564940959215164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961010214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109237439930439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961017835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961017836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961043887, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7405363321304321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961043888, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961043888, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784961051559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125059425830841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961067626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316492408514023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961083433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111152283847332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961099598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314146965742111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961099599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961099600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961125380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7381808161735535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961125380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961125380, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784961140978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11161806434392929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961156709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055787652730942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961172798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494477093219757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961180572, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961180572, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961206734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7434138655662537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961206735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961206735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784961214511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449215769767761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961230903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11264927685260773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961246815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630682855844498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961262927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036007106304169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961262928, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961262928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961289932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.742480456829071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961289933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961289933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784961305848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359034478664398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961321675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006511747837067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961337792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881495475769043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961345340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961345340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961371217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7468281388282776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961371218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961371218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784961378953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915900021791458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961395232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090788245201111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961411423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166863888502121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961426672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473746061325073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961426673, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961426673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961451007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7477802634239197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961451008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961451008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784961468201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467345058917999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961484835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11881788074970245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961501216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552406847476959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961509548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961509549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961535806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.748195230960846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961535807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961535807, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784961543821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139366552233696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961560635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142897978425026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961576780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188821494579315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961592769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302435398101807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961592770, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961592770, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961619231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7512657642364502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961619231, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961619231, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784961635025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752831935882568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961650984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693165451288223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961667040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069641262292862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961675049, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961675050, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961700913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7540103197097778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961700913, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961700914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784961708852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11562534421682358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961725131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11575483530759811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961741007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129197478294373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961757698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454580932855606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961757698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961757699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961783595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7573225498199463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961783595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961783596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784961799442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658053308725357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961815613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557404905557632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961831600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595542937517166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961839723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961839723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961865475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7572221755981445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961865476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961865476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784961873535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430456697940826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961889920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118079423904419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961905940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761182755231857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961922081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289849877357483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961922082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961922083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961949342, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7586656808853149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961949342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961949343, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784961965435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704407095909119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961981067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643982887268066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784961997277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203249007463455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962005003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962005003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962031493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7589748501777649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962031494, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962031494, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784962039469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263306647539139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962055755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184105485677719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962072202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116897314786911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962087919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408677160739899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962087920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962087920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962112378, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7618294358253479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962112379, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962112379, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784962128574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064106822013855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962145038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635402053594589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962160672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064569279551506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962169030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962169031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962194735, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7635859251022339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962194736, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962194736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784962202817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748255252838135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962219061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976746886968613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962235387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793212056159973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962252064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041644811630249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962252064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962252067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962276353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7643488049507141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962276354, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962276354, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784962292161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760503262281418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962308631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081416979432106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962324780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375484079122543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962332746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962332747, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962359071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7631798982620239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962359071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962359072, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784962366822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883816331624985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962382939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784405469894409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962398937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114327535033226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962415254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800297558307648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962415255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962415256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962441625, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7648054957389832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962441626, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962441626, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784962457929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535947978496552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962473766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167266428470612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962490644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833010077476501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962499085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962499086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962525641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671501040458679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962525641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962525641, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784962533640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468652844429016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962549711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994994044303894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962565985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439765244722366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962582382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106826364994049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962582382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962582383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962608562, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673689126968384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962608562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962608563, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784962624686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062297374010086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962641299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084912046790123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962657486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853336751461029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962665371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962665372, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962689125, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669713497161865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962689125, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962689126, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784962697563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003214120864868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962714213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473861336708069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962730663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224702000617981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962747224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283416301012039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962747225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962747225, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962773027, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673677206039429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962773027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962773028, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784962789147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006693542003632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962805778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384376883506775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962822413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086687445640564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962830502, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962830503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962855479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677550911903381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962855479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962855480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784962863613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103462278842926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962880829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938966274261475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962897538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231598258018494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962914075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123602092266083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962914076, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962914076, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962940045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695180773735046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962940046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962940046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784962956755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018175631761551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962973886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275626718997955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962990451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704202741384506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784962998742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784962998742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784963022892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697660326957703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784963022893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784963022893, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784963030864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465989053249359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963047722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761391371488571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963063965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133599489927292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963080083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671694576740265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963080084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963080084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963106485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706117630004883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963106485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963106486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784963122575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475356131792068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963138606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604254901409149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963154960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554944723844528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963163434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963163435, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963189052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703079581260681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963189052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963189053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784963197101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837376117706299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963212969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311063379049301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963229748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487981140613556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963246352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735005140304565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963246352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963246353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963272141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712681293487549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963272141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963272142, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784963288128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085343569517136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963304594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374483466148376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963320886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334064066410065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963328950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963328951, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963355165, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715346217155457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963355166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963355166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784963363332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430413484573364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963380103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278695821762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963396210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138230413198471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963412575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300146788358688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963412575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963412576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963438079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771716833114624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963438079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963438080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784963454523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726656019687653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963470647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737602412700653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963487072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160664469003677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963495005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963495006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963521465, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712503671646118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963521466, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963521466, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784963529600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101231187582016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963545751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043717175722122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963562441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611994564533234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963578975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029839962720871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963578976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963578976, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963603718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725918889045715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963603718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963603719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784963619633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748043656349182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963636884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369208455085754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963653407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592159628868103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784963661298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784963661298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed480431706.log b/recommendation_v4/rcp_logs/gbs_32768/seed480431706.log new file mode 100644 index 000000000..8ea05e45f --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed480431706.log @@ -0,0 +1,1362 @@ +:::MLLOG {"namespace": "", "time_ms": 1784867444848, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784867444848, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784867468581, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784867468582, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784867468582, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784867468582, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784867468582, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "seed", "value": 480431706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784867468583, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784867468772, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784867468773, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784868253139, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784868253140, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784868253439, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784868541256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13960187137126923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868556394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13951970636844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868570922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13930314779281616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784868578040, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868578041, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868602242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024071335792542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868602243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868602243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784868609454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394234299659729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868623561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938814401626587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868638344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13923543691635132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868652806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914842903614044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868652807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868652807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868675907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025042295455933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868675907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868675908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784868690444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13950300216674805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868704855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922733068466187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868719572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919144868850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868726649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868726649, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868750666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025946497917175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868750666, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868750667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784868758151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389952003955841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868772703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389254629611969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868787621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388626992702484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868802294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882331550121307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868802294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868802295, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868827557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027042627334595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868827558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868827558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784868841950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871249556541443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868856543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866952061653137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868871308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859710097312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868878667, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868878668, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868904777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028249621391296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868904778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868904778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784868911865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823392987251282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868927200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381521075963974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868941418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380544900894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868956166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789710402488708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784868956167, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868956168, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868983256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502922534942627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868983256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868983256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784868997322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379317194223404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869016026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780824840068817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869031328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735631108283997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869038847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869038847, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869064119, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503044068813324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869064120, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869064120, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784869071614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376889944076538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869087276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13713300228118896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869102196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727882504463196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869117903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750839233398438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869117904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869117904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869144110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031846761703491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869144111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869144111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784869159170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694360852241516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869175104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600842654705048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784869190230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371055245399475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869197605, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869197605, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869222098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033061504364014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869222099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869222099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784869229617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467338681221008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869244770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355947107076645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869259599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577140867710114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869274296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359337419271469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869274364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869274365, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869302045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503423810005188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869302046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869302046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784869316696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475078344345093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869331540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527940213680267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869347374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356479823589325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869355388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869355389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869382062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035634636878967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869382063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869382063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784869389534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601453602313995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869404787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135273277759552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869420058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289812207221985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869435723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349869668483734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869435723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869435724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869461325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038191080093384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869461325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869461326, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784869476867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422535359859467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869492392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533399999141693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869508589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514146208763123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869516409, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869516410, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869542159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5044566988945007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869542160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869542160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784869549690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627605140209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869564951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372358679771423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869580991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440744578838348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869596307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236114382743835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869596373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869596373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869622550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5052478313446045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869622551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869622551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784869637564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299736380577087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869653123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141265511512756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869668508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305820524692535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869676055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869676055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869702348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061846971511841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869702349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869702349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784869709946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382550597190857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869725159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108554482460022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869753738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996336817741394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869769221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305082142353058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869769222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869769222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869795935, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074151754379272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869795935, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869795935, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784869811139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281163573265076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869826391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984523177146912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869842539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306927651166916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869850305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869850306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869876502, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5087503790855408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869876502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869876502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784869884325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153862953186035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869899628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351822316646576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869915187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313345611095428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869931321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355888843536377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869931322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869931322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869957443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5102331042289734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869957443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869957443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784869972940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514763116836548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869988489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245455920696259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870003836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321825861930847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870011530, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870011530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870035248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5117356181144714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870035248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870035248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784870043604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641242682933807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870059818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536536693572998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870075655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620659708976746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870091507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634614646434784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870091508, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870091508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870115588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132945775985718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870115588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870115588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784870132033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479682803153992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870148273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088038563728333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870163908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362115442752838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870171947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870171948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870196996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5149398446083069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870196996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870196997, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784870204703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593539595603943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870221039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244375586509705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870236813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261418044567108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870252633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072596490383148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870252633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870252633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870279091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5163095593452454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870279092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870279092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784870294595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100048899650574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870310818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381828635931015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870326444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963172793388367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870334175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870334175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870360222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.517895519733429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870360222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870360222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784870367814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941692769527435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870383616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469967246055603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870399102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457255065441132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870415700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043828308582306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870415700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870415701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870441899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5198949575424194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870441899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870441899, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784870457581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799577414989471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870473113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187435269355774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870489023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546621799468994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870496822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870496823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870521429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5221729278564453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870521429, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870521430, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784870529278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654735684394836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870545392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358831405639648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870561313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320589780807495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870576701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090161979198456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870576701, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870576701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870602597, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248616933822632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870602598, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870602598, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784870618058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294836848974228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870633552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241124153137207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870664333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532134890556335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870672356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870672357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870698502, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5279560089111328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870698502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870698502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784870706510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319103240966797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870722407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332870602607727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870738564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494209945201874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870754363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455268740653992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870754363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870754364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870780569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5316870212554932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870780569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870780569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784870796295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590353727340698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870813127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392561674118042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870829341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309741020202637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870837335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870837336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870863066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5359281301498413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870863066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870863066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784870871390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1402277946472168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870887470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352296769618988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870903767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432475924491882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870920651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512077927589417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870920652, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870920653, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870946536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5404513478279114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870946536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870946536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784870962498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509441912174225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870978663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320186734199524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870994691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217371702194214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871002856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871002856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871030117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5453173518180847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871030118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871030118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784871038073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241620361804962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871054450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073284924030304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871069970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577063381671906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871085783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206464052200317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871085784, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871085784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871109498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5512090921401978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871109499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871109499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784871125404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443222641944885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871141908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998931109905243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871157814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317145824432373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871165678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871165679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871192047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5588834881782532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871192047, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871192048, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784871199976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203224539756775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871215787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036257028579712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871231783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13272826373577118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871247704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13223569095134735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871247705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871247705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871275502, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5679917931556702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871275502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871275502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784871291401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169607520103455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871307181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770648300647736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871322909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355958461761475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871330762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871330763, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871357143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5770664215087891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871357143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871357143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784871365215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416656851768494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871380994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12638649344444275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871397324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12626144289970398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871413742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324056088924408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871413742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871413743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871439480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5879344940185547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871439481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871439481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784871455270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285756230354309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871471179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278831660747528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871487451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274101734161377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871495172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871495172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871519611, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5939127802848816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871519611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871519611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784871527530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935884296894073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871554202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267962008714676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871570612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928639352321625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871586898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763184309005737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871586899, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871586900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871613067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6026132106781006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871613067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871613068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784871629203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12602275609970093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871645854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911652028560638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871661911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319112330675125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871669977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871669977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871694672, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6098581552505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871694672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871694672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784871703466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13022561371326447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871720003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12552836537361145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871736574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037095963954926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871752942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553226947784424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871752943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871752943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871779548, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6164672374725342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871779548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871779549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784871796022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12795931100845337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871812156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12930437922477722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871828931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12728019058704376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871837263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871837264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871863043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.622691810131073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871863043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871863043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784871871250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12660594284534454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871887529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262943148612976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871904147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266767829656601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871920533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12579941749572754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871920533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871920534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871946101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6265881657600403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871946101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871946101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784871962949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12023770064115524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871979143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11959978938102722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871994943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763253808021545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872002939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872002939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872029166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6279382705688477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872029166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872029167, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784872036982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963594704866409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872052550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677604913711548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872068509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12153321504592896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872084180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11853953450918198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872084181, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872084181, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872109089, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6378780603408813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872109090, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872109090, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784872125878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12015789747238159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872142018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148414015769958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872158117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174096167087555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872166229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872166230, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872192338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6482210755348206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872192338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872192339, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784872200175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11885418742895126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872216816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11863496899604797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872233020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131073608994484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872248737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698250472545624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872248812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872248812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872276747, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6679155230522156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872276748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872276748, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784872292580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610491573810577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872308506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11613431572914124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872324644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923086106777191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872332560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872332561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872358805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6924450993537903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872358806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872358806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784872366759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354181915521622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872383373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482144892215729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872399403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11321911215782166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872416373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503211408853531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872416374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872416374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872443267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7239167094230652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872443267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872443267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784872459484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391657590866089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872475359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11335363239049911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872491672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755858361721039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872499293, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872499294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872525739, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7435150146484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872525740, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872525740, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784872533798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759831964969635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872550140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503562331199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872566259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865181684494019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872582563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289098113775253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872582564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872582564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872609186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7529715895652771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872609187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872609187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784872625315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160656288266182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872641295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339071393013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872657561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164426803588867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872665088, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872665089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872691195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7576790452003479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872691196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872691196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784872699524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11168144643306732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872715810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132410392165184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872732452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11326690763235092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872748006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769503563642502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872748007, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872748007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872773078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592757940292358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872773079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872773079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784872790514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586843430995941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872807142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998851597309113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872823872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1166994497179985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872832137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872832138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872857968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606172561645508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872857969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872857969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784872865939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163027212023735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872882863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587199568748474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872899154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297629773616791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872915140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374258249998093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872915141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872915141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872942715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7617396116256714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872942716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872942716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784872959326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830100417137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872975593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11777079105377197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872992644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115650087594986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873000693, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873000694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873026701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7615295052528381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873026701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873026701, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784873035105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601066589355469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873051808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578478664159775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873068220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117202416062355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873085156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462786257266998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873085156, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873085157, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873111212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637561559677124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873111212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873111213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784873127378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067371815443039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873144027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063401997089386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873160307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635355114936829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873168518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873168519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873194499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7642830610275269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873194500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873194500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784873202756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425573587417603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873219577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139756441116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873235824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787396132946014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873252586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303438991308212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873252586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873252587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873279658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664561867713928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873279659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873279659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784873296303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069658100605011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873312357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635150969028473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873328988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213606059551239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873336986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873336986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873363484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670254111289978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873363485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873363485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784873371850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305430740118027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873388345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174938291311264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873405313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162330210208893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873421267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425014048814774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873421268, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873421268, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873445143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686237096786499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873445144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873445144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784873461569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659242421388626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873478140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634402930736542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873493943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711205005645752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873502288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873502288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873528126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692049145698547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873528127, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873528127, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784873536302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699662566184998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873552730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000220775604248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873569247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783572494983673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873585796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10437917709350586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873585796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873585797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873610093, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692315578460693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873610094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873610094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784873626114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789170116186142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873643222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821069777011871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873660013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443820059299469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873668073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873668074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873693404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689886689186096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873693404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873693404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784873701525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912206768989563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873718013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803283005952835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873734680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195133626461029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873751393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835914313793182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873751394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873751394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873777601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7700977921485901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873777601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873777602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784873794333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566505044698715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873810475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200470685958862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873827472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803614556789398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873836161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873836162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873862628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706372737884521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873862629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873862629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784873870930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440228879451752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873887318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965522587299347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873904023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450385302305222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873920895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096329987049103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873920896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873920896, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873947683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707976698875427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873947684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873947684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784873964264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654191672801971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873981345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904353857040405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784873997851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815221071243286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874005920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874005920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874030936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710488438606262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874030937, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874030937, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784874039789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983755439519882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874057068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480110138654709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874073974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290328204631805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874090988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294040083885193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874090989, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874090990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874117845, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705087661743164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874117845, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874117845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784874134280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997430235147476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874151297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405563354492188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874168433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079305410385132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874176681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874176682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874202240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710055112838745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874202240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874202240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784874210396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095263063907623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874227936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977377742528915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874244586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282762140035629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874261318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123505979776382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874261319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874261319, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874287693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714809775352478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874287693, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874287693, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784874304375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028919368982315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874321591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301977932453156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874338357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11692669987678528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874346790, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874346790, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874372140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771756649017334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874372141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874372141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784874380072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1149681881070137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874397251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793116688728333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874413539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014094352722168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874430245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705330967903137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874430245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874430246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874456269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723060250282288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874456269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874456269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784874472725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515276342630386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874489393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106197290122509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874506312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051708459854126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874514839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874514840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874540992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723135352134705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874540992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874540993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784874549127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861322283744812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874565689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356128215789795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874582474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488907247781754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874599268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764247179031372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874599268, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874599269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874624791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725885510444641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874624792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874624792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784874640745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100814089179039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874657201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401462763547897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874673687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362040996551514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874681738, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874681739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874708129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726647853851318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874708129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874708129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784874716525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408464074134827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874733532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321643948554993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874749804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388707906007767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874766553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347025096416473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874766553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874766553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874793587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727397084236145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874793587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874793588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784874810455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073504164814949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874826882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073383018374443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874843464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163310170173645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874851721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874851721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874877942, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728522419929504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874877942, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874877943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784874886294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191021114587784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874902815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063903033733368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874919762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061922237277031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874936476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033170223236084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874936477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874936477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874962037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773128867149353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874962038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874962038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784874978203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758358985185623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784874995850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372114181518555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875012531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587470233440399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875020758, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875020759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875047035, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732409238815308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875047036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875047036, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784875055404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351286083459854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875071754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826461017131805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875088199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026310995221138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875105092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695387423038483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875105092, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875105092, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875131068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734061479568481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875131068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875131069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784875148010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209468007087708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875164330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664685070514679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875181119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954679548740387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875189626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875189627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875215972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735728621482849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875215973, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875215973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784875224598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174821853637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875240966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717251151800156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875257848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295665264129639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875274726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287126898765564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875274726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875274727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875301164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773654580116272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875301165, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875301165, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784875317627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529692471027374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875334946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536757111549377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.631666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875351940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221002042293549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875360452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875360453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875387365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738010883331299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875387365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875387366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784875395386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073450818657875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875412576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271215230226517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875429939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911401361227036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875446878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275266855955124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875446879, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875446879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875473116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738651037216187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875473117, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875473117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784875490207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929903388023376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.831666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875507213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402737349271774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875524237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219710856676102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875532864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875532865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875558454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740628123283386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875558455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875558455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784875567121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236479133367538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875583839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181817948818207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875601221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075657457113266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784875618639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078696697950363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1.0031666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875618640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875618640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875645406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774256706237793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875645406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875645406, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784875662105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391526460647583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1.0064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875679442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874009132385254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1.0098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875696403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155892163515091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1.0131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875705253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875705254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875731648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742708921432495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875731648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875731648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784875740120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114581748843193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1.0165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875757486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381974279880524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1.0198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875773820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052607744932175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1.0231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875791233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853676497936249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1.0264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875791233, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875791234, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875817241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774607241153717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875817241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875817242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784875833902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347690433263779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1.0298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875850297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992682188749313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1.0331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875866664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020459532737732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1.0365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875875027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875875027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875900240, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747066020965576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875900241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875900241, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784875908543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09985481202602386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1.0398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875925540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087360441684723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1.0431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875941982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275264084339142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1.0464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875958943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574311017990112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1.0498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784875958944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784875958944, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784875985233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747799158096313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784875985234, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784875985234, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784876001194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453285276889801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1.0531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876018495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029452085494995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1.0565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876035394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615034401416779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1.0598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876043703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876043704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876070451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774861752986908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876070451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876070452, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784876078927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015370637178421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1.0631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876095796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646311193704605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1.0664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876112657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005607470870018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1.0698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876129224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189248621463776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1.0731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876129225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876129225, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876156018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749244570732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876156019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876156019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784876172804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196132957935333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1.0765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876190193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037810891866684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1.0798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876206424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158762335777283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1.0831666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876221957, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876221959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876248455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775108277797699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876248455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876248455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784876256577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416729003190994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1.0864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876273580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732139647006989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1.0898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876289850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241137444972992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1.0931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876306686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006992906332016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1.0965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876306686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876306687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876332461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752127647399902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876332461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876332461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784876349233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528814047574997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1.0998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876365462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017683744430542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1.1031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876382383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212449729442596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1.1065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876390691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876390692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876416805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752569317817688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876416805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876416805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784876425320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202415078878403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1.1098333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876441522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095343679189682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1.1131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876458102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077665388584137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1.1165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876475253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833108425140381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1.1198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876475254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876475254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876501610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754195332527161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876501611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876501611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784876518211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268713533878326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1.1231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876535068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478586703538895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1.1265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876551731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077724605798721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1.1298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876560126, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876560126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876586316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775762677192688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876586317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876586317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784876594752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516747087240219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1.1331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876612033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425721108913422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1.1365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876628921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712283104658127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1.1398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876646027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914191603660583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1.1431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876646028, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876646028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876671843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757933735847473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876671844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876671844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784876688372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891011357307434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1.1465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876705616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883310437202454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1.1498333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876723329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767901688814163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1.1531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876731540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876731541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876757956, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759814858436584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876757957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876757957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784876766396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125701442360878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1.1565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876783381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014118045568466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1.1598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876800640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735329985618591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1.1631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876817525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669127851724625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1.1665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876817526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876817526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876843551, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776190996170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876843552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876843552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784876860819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663922131061554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230031360, "lr": 1.1698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876877938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510078817605972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 230686720, "lr": 1.1731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876894883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113302409648895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231342080, "lr": 1.1765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876903037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876903037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876929708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761644721031189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876929709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876929709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 231669760, "epoch_num": 0.10112894085451725}} +:::MLLOG {"namespace": "", "time_ms": 1784876937993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852935165166855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 231997440, "lr": 1.1798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876955608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103091612458229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 232652800, "lr": 1.1831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876973478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081992089748383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233308160, "lr": 1.1865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876990733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117633655667305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 233963520, "lr": 1.1898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784876990734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784876990735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784877017147, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776522159576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784877017147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784877017147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 233963520, "epoch_num": 0.10213021749664118}} +:::MLLOG {"namespace": "", "time_ms": 1784877034418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444389820098877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 234618880, "lr": 1.1931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877051930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344210803508759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235274240, "lr": 1.1965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877069031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159084737300873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 235929600, "lr": 1.1998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877077827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877077828, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877104135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765971422195435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877104135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877104136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 236257280, "epoch_num": 0.10313149413876511}} +:::MLLOG {"namespace": "", "time_ms": 1784877112499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889574885368347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 236584960, "lr": 1.2031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877130111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282279342412949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237240320, "lr": 1.2064999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877146629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385565996170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 237895680, "lr": 1.2098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877164416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188768595457077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 238551040, "lr": 1.2131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877164416, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877164417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877190947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767658829689026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877190948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877190948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 238551040, "epoch_num": 0.10413277078088905}} +:::MLLOG {"namespace": "", "time_ms": 1784877208321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765376478433609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239206400, "lr": 1.2165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877225568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369530856609344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 239861760, "lr": 1.2198333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877242921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364969611167908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 240517120, "lr": 1.2231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877251771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877251772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877277340, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770736217498779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877277340, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877277340, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 240844800, "epoch_num": 0.10513404742301298}} +:::MLLOG {"namespace": "", "time_ms": 1784877285901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145663931965828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241172480, "lr": 1.2264999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877303232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527255922555923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 241827840, "lr": 1.2298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877320473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450725048780441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 242483200, "lr": 1.2331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877337390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11855706572532654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243138560, "lr": 1.2365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877337391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877337392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877362317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77735435962677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877362318, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877362318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 243138560, "epoch_num": 0.10613532406513691}} +:::MLLOG {"namespace": "", "time_ms": 1784877380326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944036394357681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 243793920, "lr": 1.2398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877397515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549168080091476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 244449280, "lr": 1.2431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877414868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143721491098404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245104640, "lr": 1.2464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877423531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877423531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877449005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777315080165863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877449005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877449005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 245432320, "epoch_num": 0.10713660070726085}} +:::MLLOG {"namespace": "", "time_ms": 1784877457594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085381358861923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 245760000, "lr": 1.2498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877474883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125624030828476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 246415360, "lr": 1.2531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877492260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080082580447197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247070720, "lr": 1.2565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877509507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776368528604507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 247726080, "lr": 1.2598333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877509507, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877509508, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877535612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775195240974426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877535613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877535613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 247726080, "epoch_num": 0.10813787734938478}} +:::MLLOG {"namespace": "", "time_ms": 1784877552378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380261927843094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 248381440, "lr": 1.2631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877569781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983745008707047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249036800, "lr": 1.2664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877587198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837839543819427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 249692160, "lr": 1.2698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877595668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877595668, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877621718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776509523391724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877621718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877621719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 250019840, "epoch_num": 0.10913915399150871}} +:::MLLOG {"namespace": "", "time_ms": 1784877630206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612668097019196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 250347520, "lr": 1.2731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877647487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492366552352905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251002880, "lr": 1.2765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877664867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454561561346054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 251658240, "lr": 1.2798333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877681796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886861383914948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252313600, "lr": 1.2831666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877681796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877681797, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877707802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777895331382751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877707803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877707803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 252313600, "epoch_num": 0.11014043063363264}} +:::MLLOG {"namespace": "", "time_ms": 1784877725088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121919751167297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 252968960, "lr": 1.2864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877742346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549067705869675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 253624320, "lr": 1.2898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877758818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815805196762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254279680, "lr": 1.2931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877767810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877767810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877794154, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780740261077881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877794155, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877794155, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 254607360, "epoch_num": 0.11114170727575658}} +:::MLLOG {"namespace": "", "time_ms": 1784877802408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068798303604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 254935040, "lr": 1.2965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877819570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067953035235405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 255590400, "lr": 1.2998333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877836203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952312499284744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256245760, "lr": 1.3031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877853219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076192557811737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 256901120, "lr": 1.3065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877853220, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877853220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877879068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782573103904724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877879068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877879068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 256901120, "epoch_num": 0.11214298391788051}} +:::MLLOG {"namespace": "", "time_ms": 1784877895926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677085071802139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 257556480, "lr": 1.3098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877912616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528752207756042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258211840, "lr": 1.3131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877929836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414434969425201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 258867200, "lr": 1.3165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877938453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877938453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877965256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778245747089386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877965256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877965256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 259194880, "epoch_num": 0.11314426056000444}} +:::MLLOG {"namespace": "", "time_ms": 1784877974004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11186493933200836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 259522560, "lr": 1.3198333333333335e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784877990612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650398582220078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260177920, "lr": 1.3231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878008028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539013892412186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 260833280, "lr": 1.3265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878025407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577202588319778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 261488640, "lr": 1.3298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878025408, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878025408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878051655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785159945487976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878051656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878051656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 261488640, "epoch_num": 0.11414553720212837}} +:::MLLOG {"namespace": "", "time_ms": 1784878068522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031506359577179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262144000, "lr": 1.3331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878085478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132347047328949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 262799360, "lr": 1.3365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878102368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498636215925217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 263454720, "lr": 1.3398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878110804, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878110804, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878136281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778559684753418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878136281, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878136281, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 263782400, "epoch_num": 0.11514681384425231}} +:::MLLOG {"namespace": "", "time_ms": 1784878144704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908696055412292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264110080, "lr": 1.3431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878162005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813461244106293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 264765440, "lr": 1.3465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878178868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484569519758224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 265420800, "lr": 1.3498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878195843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393667966127396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266076160, "lr": 1.3531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878195843, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878195843, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878219834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789802551269531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878219834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878219835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 266076160, "epoch_num": 0.11614809048637624}} +:::MLLOG {"namespace": "", "time_ms": 1784878236258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451085865497589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 266731520, "lr": 1.3565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878254300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071353480219841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 267386880, "lr": 1.359833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878271674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895939916372299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268042240, "lr": 1.3631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878280532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878280533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878306866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790228724479675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878306867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878306867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 268369920, "epoch_num": 0.11714936712850017}} +:::MLLOG {"namespace": "", "time_ms": 1784878315317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114048212766647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 268697600, "lr": 1.3665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878332417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535639524459839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 269352960, "lr": 1.3698333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878349489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365013778209686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270008320, "lr": 1.3731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878366597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961153358221054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 270663680, "lr": 1.3765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878366598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878366598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878392899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793654799461365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878392900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878392900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 270663680, "epoch_num": 0.1181506437706241}} +:::MLLOG {"namespace": "", "time_ms": 1784878410236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110635906457901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271319040, "lr": 1.379833333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878427782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084513366222382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 271974400, "lr": 1.3831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878444605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903294384479523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 272629760, "lr": 1.3865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878453024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878453025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878479048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795668244361877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878479048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878479048, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 272957440, "epoch_num": 0.11915192041274804}} +:::MLLOG {"namespace": "", "time_ms": 1784878487372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149115860462189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273285120, "lr": 1.3898333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878505552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069142818450928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 273940480, "lr": 1.3931666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878522647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795582622289658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 274595840, "lr": 1.3965e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878539842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608877032995224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275251200, "lr": 1.3998333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878539842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878539843, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878566408, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796778082847595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878566408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878566408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 275251200, "epoch_num": 0.12015319705487197}} +:::MLLOG {"namespace": "", "time_ms": 1784878583631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111283078789711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 275906560, "lr": 1.4031666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878600862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022337526082993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 276561920, "lr": 1.4065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878618721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11875507235527039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277217280, "lr": 1.4098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878627390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878627391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878653464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800881266593933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878653465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878653465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 277544960, "epoch_num": 0.1211544736969959}} +:::MLLOG {"namespace": "", "time_ms": 1784878662156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289004236459732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 277872640, "lr": 1.4131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878679158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954069346189499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 278528000, "lr": 1.4165000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878696945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747020483016968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279183360, "lr": 1.4198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878714419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951782017946243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 279838720, "lr": 1.4231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878714420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878714421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878741254, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802106142044067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878741254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878741254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 279838720, "epoch_num": 0.12215575033911984}} +:::MLLOG {"namespace": "", "time_ms": 1784878758283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112889647483826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 280494080, "lr": 1.4265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878775584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212456226348877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281149440, "lr": 1.4298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878792913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11575989425182343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 281804800, "lr": 1.4331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878801610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878801611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878827995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803984880447388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878827995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878827995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 282132480, "epoch_num": 0.12315702698124378}} +:::MLLOG {"namespace": "", "time_ms": 1784878836816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680616647005081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 282460160, "lr": 1.4365000000000001e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878854711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350098252296448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283115520, "lr": 1.4398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878871989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177115142345428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 283770880, "lr": 1.4431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878889004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131538450717926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 284426240, "lr": 1.4464999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878889005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878889005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878916006, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780606746673584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878916007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878916007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 284426240, "epoch_num": 0.12415830362336772}} +:::MLLOG {"namespace": "", "time_ms": 1784878933218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864861309528351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285081600, "lr": 1.4498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878950482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331866681575775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 285736960, "lr": 1.4531666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878967887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138535737991333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 286392320, "lr": 1.4564999999999997e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784878976298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784878976299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784879001716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806305289268494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784879001717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784879001717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 286720000, "epoch_num": 0.12515958026549165}} +:::MLLOG {"namespace": "", "time_ms": 1784879010343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447151005268097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287047680, "lr": 1.4598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879027620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074212193489075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 287703040, "lr": 1.4631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879045382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306057870388031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 288358400, "lr": 1.4664999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879062679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345760524272919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289013760, "lr": 1.4698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879062679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879062680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879089182, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809916734695435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879089182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879089182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 289013760, "epoch_num": 0.12616085690761558}} +:::MLLOG {"namespace": "", "time_ms": 1784879106286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772402584552765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 289669120, "lr": 1.4731666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879123172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544408857822418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290324480, "lr": 1.4764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879141365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10091269016265869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 290979840, "lr": 1.4798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879150053, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879150053, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879176014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807896137237549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879176015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879176015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 291307520, "epoch_num": 0.12716213354973951}} +:::MLLOG {"namespace": "", "time_ms": 1784879184593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580084472894669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 291635200, "lr": 1.4831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879201809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873007774353027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292290560, "lr": 1.4864999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879219042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485218465328217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 292945920, "lr": 1.4898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879236178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538893193006516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 293601280, "lr": 1.4931666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879236179, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879236180, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879262241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805507779121399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879262241, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879262242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 293601280, "epoch_num": 0.12816341019186345}} +:::MLLOG {"namespace": "", "time_ms": 1784879279452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802070796489716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294256640, "lr": 1.4964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879296813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394807159900665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 294912000, "lr": 1.4998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879314628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781040787696838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 295567360, "lr": 1.5031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879323449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879323449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879350842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808706164360046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879350843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879350843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 295895040, "epoch_num": 0.12916468683398738}} +:::MLLOG {"namespace": "", "time_ms": 1784879359400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041443794965744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296222720, "lr": 1.5064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879375915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051386296749115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 296878080, "lr": 1.5098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879393681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876233875751495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 297533440, "lr": 1.5131666666666668e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879410996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965332388877869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298188800, "lr": 1.5164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879410996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879410997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879437607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808049917221069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879437607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879437607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 298188800, "epoch_num": 0.1301659634761113}} +:::MLLOG {"namespace": "", "time_ms": 1784879454977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519455373287201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 298844160, "lr": 1.5198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879472062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322828590869904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 299499520, "lr": 1.5231666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879489246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389071702957153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300154880, "lr": 1.5265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879497741, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879497742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879524243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810112237930298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879524243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879524243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 300482560, "epoch_num": 0.13116724011823525}} +:::MLLOG {"namespace": "", "time_ms": 1784879532917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041580885648727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 300810240, "lr": 1.5298333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879549837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000039279460907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 301465600, "lr": 1.5331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879567324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109160840511322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302120960, "lr": 1.5364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879584754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718171298503876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 302776320, "lr": 1.5398333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879584755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879584755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879611101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780792772769928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879611101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879611101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 302776320, "epoch_num": 0.13216851676035918}} +:::MLLOG {"namespace": "", "time_ms": 1784879627786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989843308925629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 303431680, "lr": 1.5431666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879645255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079757958650589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304087040, "lr": 1.5465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879662728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371625423431396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 304742400, "lr": 1.5498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879671243, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879671244, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879698118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811894416809082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879698118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879698118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 305070080, "epoch_num": 0.1331697934024831}} +:::MLLOG {"namespace": "", "time_ms": 1784879706742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205487161874771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 305397760, "lr": 1.5531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879723959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760756582021713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306053120, "lr": 1.5564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879740961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541243851184845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 306708480, "lr": 1.5598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879757877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543207824230194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 307363840, "lr": 1.5631666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879757877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879757878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879784204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813134789466858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879784204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879784205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 307363840, "epoch_num": 0.13417107004460704}} +:::MLLOG {"namespace": "", "time_ms": 1784879801471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665886104106903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308019200, "lr": 1.5665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879818636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098711296916008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 308674560, "lr": 1.5698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879835595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422119498252869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309329920, "lr": 1.5731666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879844223, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879844224, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879869918, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812719345092773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879869919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879869919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 309657600, "epoch_num": 0.13517234668673098}} +:::MLLOG {"namespace": "", "time_ms": 1784879878662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782726854085922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 309985280, "lr": 1.5764999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879894942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205455869436264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 310640640, "lr": 1.5798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879912651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922025889158249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311296000, "lr": 1.5831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879930285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817717015743256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 311951360, "lr": 1.5865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879930286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879930286, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879956640, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7817267775535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879956641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879956641, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 311951360, "epoch_num": 0.1361736233288549}} +:::MLLOG {"namespace": "", "time_ms": 1784879973810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596449673175812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 312606720, "lr": 1.5898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784879990873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644502937793732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313262080, "lr": 1.5931666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880008335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489962995052338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 313917440, "lr": 1.5964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880016431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880016432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880043190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820276618003845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880043191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880043191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 314245120, "epoch_num": 0.13717489997097884}} +:::MLLOG {"namespace": "", "time_ms": 1784880051944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890327394008636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 314572800, "lr": 1.5998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880068996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426285862922668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315228160, "lr": 1.6031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880086367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358267277479172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 315883520, "lr": 1.6065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880103353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264913737773895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 316538880, "lr": 1.6098333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880103354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880103354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880129711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818729877471924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880129711, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880129711, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 316538880, "epoch_num": 0.13817617661310277}} +:::MLLOG {"namespace": "", "time_ms": 1784880147477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759764909744263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317194240, "lr": 1.6131666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880164278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093674898147583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 317849600, "lr": 1.6165e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880181880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265158653259277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 318504960, "lr": 1.6198333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880190711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880190712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880217487, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825738191604614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880217487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880217487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 318832640, "epoch_num": 0.1391774532552267}} +:::MLLOG {"namespace": "", "time_ms": 1784880226461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147259995341301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319160320, "lr": 1.6231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880243685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432347655296326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 319815680, "lr": 1.6265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880261231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272400617599487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 320471040, "lr": 1.6298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880278734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081160306930542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321126400, "lr": 1.6331666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880278735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880278735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880304539, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7824981212615967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880304539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880304540, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 321126400, "epoch_num": 0.14017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784880321708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110304519534111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 321781760, "lr": 1.6365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880339404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878812521696091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 322437120, "lr": 1.6398333333333331e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880356900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272354423999786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323092480, "lr": 1.6431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880365752, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880365753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880391595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7824715375900269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880391596, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880391596, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 323420160, "epoch_num": 0.14118000653947457}} +:::MLLOG {"namespace": "", "time_ms": 1784880400191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932572185993195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 323747840, "lr": 1.6465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880417742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488694906234741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 324403200, "lr": 1.6498333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880434518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581450164318085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325058560, "lr": 1.6531666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880452609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382389068603516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 325713920, "lr": 1.6565e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880452610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880452610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880478826, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831377983093262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880478826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880478826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 325713920, "epoch_num": 0.1421812831815985}} +:::MLLOG {"namespace": "", "time_ms": 1784880496200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993654280900955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 326369280, "lr": 1.6598333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880513561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119922548532486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327024640, "lr": 1.6631666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880531010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788845270872116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680000, "lr": 1.6665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880539815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880539815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880566864, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831099629402161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880566864, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880566864, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 328007680, "epoch_num": 0.14318255982372244}} +:::MLLOG {"namespace": "", "time_ms": 1784880575271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398866027593613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328335360, "lr": 1.6698333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880592647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081230565905571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 328990720, "lr": 1.6731666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880610079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127447560429573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 329646080, "lr": 1.6765e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880627860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886513441801071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330301440, "lr": 1.6798333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880627862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880627863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880654587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834937572479248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880654587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880654588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 330301440, "epoch_num": 0.14418383646584637}} +:::MLLOG {"namespace": "", "time_ms": 1784880671390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487838625907898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 330956800, "lr": 1.6831666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880689465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120140478014946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 331612160, "lr": 1.6865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880707067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319690406322479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332267520, "lr": 1.6898333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880715569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880715570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880741602, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828986048698425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880741602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880741602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 332595200, "epoch_num": 0.1451851131079703}} +:::MLLOG {"namespace": "", "time_ms": 1784880750391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714592039585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 332922880, "lr": 1.6931666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880767532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414561629295349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 333578240, "lr": 1.6964999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880785047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080765575170517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334233600, "lr": 1.6998333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880802143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169827938079834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 334888960, "lr": 1.7031666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880802143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880802144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880828355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832822799682617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880828356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880828356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 334888960, "epoch_num": 0.14618638975009424}} +:::MLLOG {"namespace": "", "time_ms": 1784880845532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052163764834404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 335544320, "lr": 1.7064999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880862906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674464702606201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336199680, "lr": 1.7098333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880879856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845504701137543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 336855040, "lr": 1.7131666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880888538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880888539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880914893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839930057525635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880914894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880914894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 337182720, "epoch_num": 0.14718766639221817}} +:::MLLOG {"namespace": "", "time_ms": 1784880923832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997718781232834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 337510400, "lr": 1.7164999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880940564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220523923635483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338165760, "lr": 1.7198333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880958151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721482336521149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 338821120, "lr": 1.7231666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880975986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241539031267166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 339476480, "lr": 1.7265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784880975987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784880975988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784881002090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7838149070739746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784881002091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784881002091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 339476480, "epoch_num": 0.1481889430343421}} +:::MLLOG {"namespace": "", "time_ms": 1784881019251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017722561955452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340131840, "lr": 1.7298333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881036587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357986390590668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 340787200, "lr": 1.7331666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881053855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623490065336227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 341442560, "lr": 1.7364999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881062526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881062527, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881089007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7844852209091187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881089007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881089007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 341770240, "epoch_num": 0.14919021967646603}} +:::MLLOG {"namespace": "", "time_ms": 1784881097331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348066687583923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342097920, "lr": 1.7398333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881114317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013488695025444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 342753280, "lr": 1.7431666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881131527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627192258834839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 343408640, "lr": 1.7465e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881148751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817524790763855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344064000, "lr": 1.7498333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881148751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881148752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881175306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840684652328491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881175306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881175307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 344064000, "epoch_num": 0.15019149631858997}} +:::MLLOG {"namespace": "", "time_ms": 1784881192869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608062148094177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 344719360, "lr": 1.7531666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881209495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792084038257599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 345374720, "lr": 1.7564999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881226202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998866930603981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346030080, "lr": 1.7598333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881235339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881235339, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881261942, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832533121109009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881261942, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881261942, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 346357760, "epoch_num": 0.1511927729607139}} +:::MLLOG {"namespace": "", "time_ms": 1784881270719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408343374729156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 346685440, "lr": 1.7631666666666667e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881288014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297829657793045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347340800, "lr": 1.7665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881305626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067451760172844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 347996160, "lr": 1.7698333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881322648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133975744247437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 348651520, "lr": 1.7731666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881322715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881322716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881349248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845158576965332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881349248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881349249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 348651520, "epoch_num": 0.15219404960283783}} +:::MLLOG {"namespace": "", "time_ms": 1784881366468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860717296600342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349306880, "lr": 1.7764999999999998e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881383848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718584060668945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 349962240, "lr": 1.7798333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881400820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421346873044968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 350617600, "lr": 1.7831666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881409630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881409631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881436001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7842366099357605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881436001, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881436001, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 350945280, "epoch_num": 0.15319532624496177}} +:::MLLOG {"namespace": "", "time_ms": 1784881444717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047084778547287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351272960, "lr": 1.7865e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881462126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063154250383377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 351928320, "lr": 1.7898333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881479442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015848278999329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 352583680, "lr": 1.7931666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881495929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551470518112183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353239040, "lr": 1.7964999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881495995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881495996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881522896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7844433188438416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881522897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881522897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 353239040, "epoch_num": 0.1541966028870857}} +:::MLLOG {"namespace": "", "time_ms": 1784881540965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625308752059937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 353894400, "lr": 1.7998333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881558599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574117302894592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 354549760, "lr": 1.8031666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881575664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595713555812836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355205120, "lr": 1.8065e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881584311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881584312, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881610972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7853514552116394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881610972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881610972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 355532800, "epoch_num": 0.15519787952920963}} +:::MLLOG {"namespace": "", "time_ms": 1784881619154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340135544538498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 355860480, "lr": 1.8098333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881636598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204973816871643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 356515840, "lr": 1.8131666666666664e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881653798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213686525821686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357171200, "lr": 1.8164999999999999e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881670892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727822035551071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 357826560, "lr": 1.8198333333333333e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881670893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881670893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881697143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783963143825531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881697144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881697144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 357826560, "epoch_num": 0.15619915617133356}} +:::MLLOG {"namespace": "", "time_ms": 1784881714591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117511451244354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 358481920, "lr": 1.8231666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881731898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207109153270721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359137280, "lr": 1.8265e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881749495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299451649188995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 359792640, "lr": 1.8298333333333332e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881758044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881758045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881784190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837881445884705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881784191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881784191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 360120320, "epoch_num": 0.1572004328134575}} +:::MLLOG {"namespace": "", "time_ms": 1784881792379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535009205341339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 360448000, "lr": 1.8331666666666665e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881809809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834812372922897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361103360, "lr": 1.8365e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881827797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832902044057846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 361758720, "lr": 1.8398333333333334e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881845205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049330979585648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 362414080, "lr": 1.8431666666666666e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784881845206, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784881845206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784881871879, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7849831581115723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784881871879, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} +:::MLLOG {"namespace": "", "time_ms": 1784881871879, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 362414080, "epoch_num": 0.15820170945558143}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed526341580.log b/recommendation_v4/rcp_logs/gbs_32768/seed526341580.log new file mode 100644 index 000000000..d4a15353b --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed526341580.log @@ -0,0 +1,666 @@ +:::MLLOG {"namespace": "", "time_ms": 1784909131269, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784909131269, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784909158466, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "seed", "value": 526341580, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784909158469, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784909158479, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784909158479, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784909811615, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784909811617, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784909811821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784910278396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873390853405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910293682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864240050315857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910308379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876973092556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910315667, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910315667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910360129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013054013252258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910360130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910360130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910367427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871541619300842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910382023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875891268253326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910396934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386837661266327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910411791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873231410980225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910411791, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910411792, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910435353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014687776565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910435353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910435353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910450105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853761553764343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910464939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847529888153076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910479995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836190104484558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910487336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910487336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910511106, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016574859619141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910511107, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910511107, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910518911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857872784137726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910533895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383584886789322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910549106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138438418507576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910564119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807858526706696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910564120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910564120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910586861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018885135650635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910586861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910586862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910601772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382719874382019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910616612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776469230651855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910631510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380549818277359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910638910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910638910, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910661736, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021727085113525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910661737, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910661737, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910669033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745126128196716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910684434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741210103034973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910699326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710175454616547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910714574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370881050825119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910714575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910714575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910738143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024818181991577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910738144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910738144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910752651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371547132730484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910767623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722294569015503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910783395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646341860294342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910790942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910790943, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910814530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028282999992371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910814531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910814531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910822160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372247040271759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910837998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634926080703735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910853156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661357760429382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910869177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370188593864441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910869177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910869178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910892172, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032877326011658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910892172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910892173, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910907234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362975835800171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910923063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502223789691925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910938038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654518127441406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784910945576, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910945577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910968780, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037441849708557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910968781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910968781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910976309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310140371322632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784910991861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13452330231666565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911006772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134946808218956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911021737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522250950336456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911021805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911021806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911045439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042166709899902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911045440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911045440, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911060305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379254937171936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911075216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462728261947632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911090166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352236568927765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911098221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911098222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911121186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047289133071899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911121187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911121187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911128492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561874628067017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911143401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480150699615479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911158637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318969875574112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911174305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458195328712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911174306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911174306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911197249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053913593292236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911197250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911197250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911212674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378125429153442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911228388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353323757648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911244330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509857654571533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911252140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911252140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911275212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064204931259155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911275213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911275213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911282880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136510968208313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911298349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334608793258667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911314318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134331613779068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911329783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214366137981415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911329846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911329847, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911352566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5075273513793945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911352567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911352567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911367957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288283348083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911384077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125059008598328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911399168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311518728733063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911406825, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911406825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911429179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086913108825684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911429179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911429179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911436722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842891156673431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911452140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109584152698517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911468071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983228266239166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911483534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036738336086273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911483534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911483535, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911506609, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100205540657043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911506610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911506610, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911522050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327946037054062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911537717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975890934467316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911553525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306544840335846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911560977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911560977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911583234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114631652832031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911583235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911583235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911590805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149988651275635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911605919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521160185337067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911621624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317516446113586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911637803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566076755523682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911637804, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911637804, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911660285, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5130100250244141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911660286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911660286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911675816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351589411497116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911691557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262131810188293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911707499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332358568906784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911715377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911715377, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911738506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.514509379863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911738507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911738507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911746935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635677099227905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911762768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352347582578659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911778554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361624300479889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911794622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364419013261795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911794622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911794623, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911818247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160394906997681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911818248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911818248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911834904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478237390518188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911851267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085666298866272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911867383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362246423959732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911875675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911875676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911899325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5175673365592957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911899325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911899325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911907274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591983914375305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911923571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252270221710205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911939287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260261714458466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911954824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087081909179688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911954824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911954825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911977477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5189923644065857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911977478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911977478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911993072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310516744852066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912009091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383226215839386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912025228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954698503017426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912033226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912033226, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912056246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5206571221351624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912056246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912056246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912064041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922954559326172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912080049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13480716943740845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912096003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462767004966736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912112568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036905229091644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912112569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912112569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912135155, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5226723551750183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912135156, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912135156, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912150760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280834674835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912166387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184398412704468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912182152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543814420700073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912190178, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912190178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912213899, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248515605926514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912213900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912213900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912221871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365807056427002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912238058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352788984775543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912253927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320158839225769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912269491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091513514518738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912269492, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912269492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912293036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5273414850234985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912293037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912293037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912308726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954849004745483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912324386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324179768562317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912340717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541127741336823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912348779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912348780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912371436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5303677916526794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912371437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912371437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912379355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318598985671997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912395452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340581953525543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912411354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496533036231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912427399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458910584449768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912427399, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912427399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5338490605354309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912466974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135977104306221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912483361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381677865982056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912499593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331564486026764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912507528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912507529, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912531261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5379123687744141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912531262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912531262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912539512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14018258452415466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912555803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521096110343933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912572134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429218530654907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912588936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501933217048645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912588936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912588936, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912612271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5422119498252869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912612272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912612272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912628450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351666897535324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912644858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133146271109581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912661365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233748078346252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912669889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912669889, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912693174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.546717643737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912693174, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912693174, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912701314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244116306304932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912717132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075481355190277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912732887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357807219028473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912748977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222664594650269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912748978, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912748978, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912772189, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5519312024116516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912772190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912772190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912788232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455882668495178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912804821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010267913341522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912820954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166996836662292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912829056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912829056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912851725, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5587932467460632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912851726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912851726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912859639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215193152427673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912875903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051161170005798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912892276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281315565109253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912908631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321818232536316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912908631, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912908632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912932622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5673744082450867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912932623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912932623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912948369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13164040446281433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912964229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277942657470703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912979817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336183249950409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912987862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912987863, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913011241, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5762316584587097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913011242, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913011242, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913019236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344124674797058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913035076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263229250907898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913051367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12643089890480042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913067499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243059813976288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913067500, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913067500, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913090582, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.587087094783783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913090583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913090583, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913106495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12888163328170776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913122361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809208035469055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913138519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273798793554306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913146373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913146374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913169693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5938713550567627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913169694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913169694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913177826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294785588979721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913194215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691166996955872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913210253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12947508692741394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913226103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12781907618045807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913226104, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913226104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913250252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.603882908821106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913250253, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913250253, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913266104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261768639087677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913282255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12908634543418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913298232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321074664592743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913306238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913306239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913330897, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6131171584129333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913330898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913330898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913339504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052353262901306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913356205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12574172019958496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913372689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047248125076294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913388884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12572461366653442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913388885, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913388885, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913412017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6233498454093933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913412018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913412018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913428581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12819552421569824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913444939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295987218618393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913461744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12720227241516113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913470086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913470086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913493793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6340071558952332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913493794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913493794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913501846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659072875976562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913517895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12642589211463928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913534310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686066329479218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913550629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12575215101242065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913550630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913550630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913574463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.642676830291748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913574464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913574464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913591017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12005706131458282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913607266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11964701861143112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913623248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11774566769599915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913631219, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913631220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913654853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6444427371025085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913654853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913654854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913662774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11949822306632996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913678967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646667867898941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913695326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114882469177246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913711496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870183795690536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913711497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913711497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913735711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6650964617729187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913735712, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913735712, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913752339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983343213796616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913768385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127642542123795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913784309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172519326210022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913792229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913792229, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913815337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6855701208114624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913815337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913815337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913823238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791616678237915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913839494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783672869205475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913855833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245064437389374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913871616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11625031381845474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913871693, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913871693, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913895569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7129432559013367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913895570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913895570, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913912122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517942696809769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913928254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510850489139557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913944568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820861905813217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913952620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913952620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913976267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7311861515045166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913976268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913976268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913984355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125912144780159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914000847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11333094537258148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914017140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153963208198547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914033459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347754299640656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914033460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914033460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914056984, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7551345825195312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914056985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914056985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914073103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223593354225159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914089215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132904887199402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914105338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550319194793701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914113341, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914113342, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914136847, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7616064548492432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914136848, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914136848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914144661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047828197479248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914161009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282044649124146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914177164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666032135486603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914193456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066770553588867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914193456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914193457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914217394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766037106513977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914217395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914217395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914233588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379291862249374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914249552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105940192937851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914265715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092788577079773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914273649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914273650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914296308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7675557732582092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914296309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914296309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914304461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992272198200226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914320862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108363419771194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914337543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158545315265656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914353549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519761592149734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914353550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914353550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914377073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692939043045044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914377074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914377074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914394386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460816115140915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914410610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887967586517334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914427031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556969583034515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914435158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914435158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914457883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704788446426392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914457884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914457884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914465994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488867551088333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914482652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435891687870026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914499037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175854504108429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914515310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312663555145264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914515311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914515311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914539586, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709818482398987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914539587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914539587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914556470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11749798059463501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914573080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11741326749324799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914589907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032170057296753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914598204, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914598205, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914621648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715117931365967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914621648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914621649, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914630084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472757905721664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914646988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11493808031082153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914663664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099553108215332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914680638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390940308570862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914680638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914680639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914704359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717825770378113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914704360, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914704360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914720078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623440146446228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914736153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546213388442993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914752202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546649992465973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914760226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914760227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914783881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723501920700073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914783882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914783882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914792027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362373292446136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914808695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091048270463943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914825081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725167393684387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914841689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227555572986603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914841689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914841690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914865480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726418375968933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914865481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914865481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914881990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064562052488327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914898205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613948106765747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914914667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156828910112381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914922706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914922706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914946843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729508280754089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914946844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914946844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914955065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236550867557526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914971553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122734308242798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914988281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083105206489563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915004500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370521247386932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915004501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915004501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915029364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730721235275269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915029365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915029365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915045937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554439574480057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915062427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619352757930756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915078496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640248656272888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915086878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915086878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915110160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77329021692276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915110160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915110161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915118414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068853810429573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915134594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09914456307888031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915151212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752206295728683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915167797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394388437271118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915167797, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915167798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915191179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734025120735168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915191181, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915191181, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915207260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732076317071915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915224108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787960886955261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915240641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399608314037323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915248813, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915248814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915272107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773574709892273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915272108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915272108, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915280147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869725793600082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915296661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702766478061676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915313044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135011911392212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915329821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786730796098709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915329821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915329822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915353177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737813591957092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915353178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915353178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915370040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482782870531082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915386455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138830542564392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915403365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789947956800461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915411743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915411744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915435416, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737798690795898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915435416, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915435417, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915443376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10406952351331711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915459406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918810427188873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915475663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137312799692154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915491903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043062806129456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915491904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915491904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915515302, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739419937133789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915515303, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915515303, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915531722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619118064641953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915548561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836128145456314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915564994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798825323581696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915573025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915573026, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915597164, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740586400032043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915597165, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915597165, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915606069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978671908378601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915623148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438559740781784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915640171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187699437141418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915657052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274714022874832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915657052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915657053, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915681633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741419076919556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915681634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915681634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915698297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975994169712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915715279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341701447963715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915732331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037234961986542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915740751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915740752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915764718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743661999702454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915764719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915764719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915772981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038659512996674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915790168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092933714389801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915806808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124717965722084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915823496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068357527256012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915823496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915823497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915847198, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774531900882721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915847199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915847199, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915863938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979249328374863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915881064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266990005970001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915898131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167503148317337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915906599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915906599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915930927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746140360832214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915930928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915930928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915939121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445701122283936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915956058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762406140565872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915972405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010226309299469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915988697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659899562597275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915988698, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915988698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916012810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744200825691223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916012811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916012811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916029174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469157248735428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916045565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591776669025421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916062314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500170290470123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916070751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916070751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916094582, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746195197105408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916094583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916094583, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916102840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824399441480637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916118918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031327024102211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916135787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475759208202362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916152245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707424581050873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916152245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916152246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916175614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746298909187317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916175615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916175615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916191744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029705613851547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916208227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350552201271057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916224560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295145958662033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916232679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916232679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916256237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747148871421814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916256238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916256238, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916264521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343869775533676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916281502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277315229177475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916297994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11326272040605545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916314720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286329686641693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916314721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916314721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916338758, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774863600730896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916338759, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916338759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916355336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692591965198517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed591489896.log b/recommendation_v4/rcp_logs/gbs_32768/seed591489896.log new file mode 100644 index 000000000..caed79f26 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed591489896.log @@ -0,0 +1,684 @@ +:::MLLOG {"namespace": "", "time_ms": 1784916346019, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784916346019, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784916369547, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "seed", "value": 591489896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784916369554, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784916369574, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784916369574, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784917296632, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784917296632, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784917296999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784917511961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916730880737305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917527030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390719711780548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917541111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891232013702393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917548258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917548259, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917590838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022097826004028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917590838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917590838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917597860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898299634456635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917611828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890250027179718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917626343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879577815532684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917640713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882529735565186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917640714, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917640714, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917662924, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023571848869324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917662924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917662924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917677367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886931538581848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917691632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386762410402298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917706022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860508799552917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917713014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917713015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917735963, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502550482749939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917735963, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917735963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917743247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385640799999237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917757575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850364089012146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917772268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854815065860748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917786710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842178881168365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917786711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917786712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917809356, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027703046798706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917809357, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917809357, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917823675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833904266357422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917838052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381373256444931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917852406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382337510585785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917859510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917859511, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917882201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030468702316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917882201, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917882202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917889196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771390914916992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917903913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776889443397522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917917881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752096891403198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917932302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749350607395172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917932303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917932303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917956428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033438801765442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917956428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917956429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917970390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375231295824051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917984892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375395953655243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917999924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705846667289734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918007170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918007170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918030184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036787986755371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918030184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918030185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918037504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375596672296524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918052869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366693377494812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918067601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694778084754944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918083043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734754920005798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918083044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918083044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918106176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5040976405143738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918106177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918106177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918120814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664434850215912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918136090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356474906206131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918150780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13680949807167053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918158122, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918158123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918181718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045098066329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918181719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918181719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918189003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398294150829315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918203935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352120190858841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918218190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541993498802185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918232585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567164540290833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918232650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918232651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918256556, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049562454223633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918256557, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918256557, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918271064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435296714305878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918285624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497979938983917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918300251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353917270898819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918308192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918308193, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918331231, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054174065589905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918331232, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918331232, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918338316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590773940086365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918352697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351233869791031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918367553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325761377811432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918382965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347222626209259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918382966, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918382966, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918405401, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060067176818848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918405402, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918405402, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918420152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401328027248383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918435332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529491424560547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918450936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512276113033295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918458583, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918458583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918481671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5069608092308044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918481672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918481672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918489035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644595444202423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918504037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353875279426575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918519247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343558132648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918533836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214752078056335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918533907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918533907, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918557387, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.508042573928833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918557388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918557388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918572022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132987380027771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918587497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312776803970337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918602030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331861913204193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918609322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918609322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918632823, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5092653632164001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918632824, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918632824, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918640103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826918601989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918654858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108892738819122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918670057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299649178981781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918684827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050684332847595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918684827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918684828, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918708142, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5106553435325623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918708143, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918708143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918722908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280485570430756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918737885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974224984645844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918753316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306486874818802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918760587, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918760587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918783901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5121439099311829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918783902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918783902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918791270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149522244930267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918805811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512638211250305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918821007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331591159105301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918836756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561254739761353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918836757, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918836757, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918859814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138207674026489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918859815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918859815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918874687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135186105966568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918889652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261359930038452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918904829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313299417495728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918912305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918912305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918935337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5155209898948669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918935338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918935338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918943757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638006150722504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918958948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519078493118286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918974225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622717559337616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918989866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364145129919052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918989866, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918989867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919013414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5172511339187622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919013415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919013415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919029433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348608285188675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919044998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085421919822693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919060383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611820340156555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919068252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919068253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919091566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5189443230628967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919091567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919091567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919099261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597983121871948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919115298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242048025131226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919130280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325603574514389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919145155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089323043823242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919145155, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919145156, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919167418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5204777717590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919167419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919167419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919182475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096502423286438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919198007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813605904579163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919213126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294737607240677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919220616, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919220617, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919243437, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5223151445388794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919243438, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919243438, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919250871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12932005524635315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919266101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134637251496315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919281457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462108373641968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919297611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032016158103943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919297611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919297612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919320416, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5245875120162964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919320417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919320417, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919335210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128128781914711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919350110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317647099494934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919365154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537120819091797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919372874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919372875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919396190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5270369648933411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919396190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919396191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919403809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365167200565338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919419507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358516991138458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919434426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195493817329407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919449239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090765476226807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919449240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919449240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919472433, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5299233794212341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919472434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919472434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919487521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12950873374938965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919502636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324438452720642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919518577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522373139858246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919526227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919526228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919549298, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5334120392799377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919549299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919549299, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919556732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318831443786621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919571804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332702934741974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919586715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485264778137207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919601909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344153732061386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919601910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919601910, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919625603, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5373625755310059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919625603, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919625604, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919640891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358959823846817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919656838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387374579906464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919672429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331101357936859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919680051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919680052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919703528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.54181307554245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919703529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919703529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919711468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14008228480815887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919727244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518056273460388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919743417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429400324821472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919759147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350385546684265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919759148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919759148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919782344, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5466248393058777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919782345, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919782345, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919797702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351310759782791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919813132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312698900699615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919828622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216888904571533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919836881, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919836882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919860686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5519147515296936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919860687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919860687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919868437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322692334651947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919883703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13057881593704224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919898762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558882474899292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919914241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321221888065338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919914241, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919914242, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919937972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5582266449928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919937972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919937973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919953533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432665169239044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919969416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986387312412262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919984827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163453340530396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919992328, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919992328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920014804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5661414861679077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920014805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920014805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920022192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319153606891632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920037468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025036454200745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920052973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326451152563095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920068974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215847313404083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920068975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920068975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920092223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5756365656852722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920092224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920092224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920107618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151895999908447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920122819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740473449230194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920137765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334744691848755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920145593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920145593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920168070, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5855426788330078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920168071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920168071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920175884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402381539344788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920191329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12617778778076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920207247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613697350025177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920222595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202089071273804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920222595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920222596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920245140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5954077243804932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920245141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920245141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920260257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12844766676425934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920275282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758806347846985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920290762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691405415534973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920298270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920298271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920321209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6034537553787231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920321210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920321210, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920328978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926989793777466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920345022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12611024081707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920360463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289048194885254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920375749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719744443893433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920375750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920375750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920398961, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6122161149978638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920398961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920398962, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920414113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12548762559890747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920429848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285838931798935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920445251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13126501441001892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920453002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920453002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920476420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6197348237037659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920476421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920476421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920484745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970620393753052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920500631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249862015247345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920516429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976869940757751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920531928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12466605007648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920531929, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920531929, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920555144, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6270090937614441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920555144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920555144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920571009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12739863991737366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920586711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871570885181427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920602973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659290432929993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920610931, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920610931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920633600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6341928839683533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920633600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920633600, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920641444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601496279239655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920656898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261768341064453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920672675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606967985630035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920688524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12542593479156494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920688524, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920688524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920711448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6398618817329407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920711449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920711449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920727692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11918236315250397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920743344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11902447044849396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920758782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699742078781128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920766504, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920766504, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920788868, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6426985263824463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920788869, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920788869, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920796369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934545636177063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920811872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11612818390130997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920827537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207909807562828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920842945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841994524002075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920842946, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920842946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920866703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6538046002388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920866703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920866703, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920882919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199551448225975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920898007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138162016868591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920913221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733553558588028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920920703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920920703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920943664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6651466488838196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920943665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920943665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920951093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11793258041143417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920966526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827854812145233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920981928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251508444547653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920997029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679022014141083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920997102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920997102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921021629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6804744005203247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921021629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921021630, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921037091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549985408782959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921052386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576216667890549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921067823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087101474404335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921075431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921075431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921098336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6998255252838135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921098337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921098337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921105997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272069811820984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921121589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481953412294388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921137136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294098198413849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921152923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502503603696823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921152924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921152924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921176262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7280968427658081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921176262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921176262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921191569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397753655910492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921206752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320644617080688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921222020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755678057670593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921229494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921229495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921252031, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7400047779083252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921252032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921252032, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921259569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710929334163666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921275191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489466577768326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921290625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854531079530716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921306313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345715820789337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921306314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921306314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921330087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7513070106506348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921330088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921330088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921345836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11650295555591583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921361171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398591101169586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921376490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118784099817276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921384022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921384023, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921406910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560759782791138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921406911, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921406911, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921414658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220357567071915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921430191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424423009157181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921446106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386209726333618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921461380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880579799413681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921461380, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921461381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921485348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606480717658997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921485349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921485349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921502029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693304777145386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921517580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12085547298192978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921533425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832462251186371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921541203, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921541203, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921563790, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7633547186851501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921563791, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921563791, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921571474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733946204185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921587432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11697021126747131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921603340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430242657661438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921619056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469586193561554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921619057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921619057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921643140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7650331854820251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921643140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921643141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921659088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975046992301941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921674702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11825934052467346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921690521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169691383838654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921698459, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921698459, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921721675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662556171417236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921721675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921721675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921729707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698329448699951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921745768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11571358144283295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921761658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207791417837143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921778067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051856130361557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921778068, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921778068, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921800904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.76688551902771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921800905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921800905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921816047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723432898521423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921831600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661903023719788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921846985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676994919776917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921854748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921854749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921878789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676841616630554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921878789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921878789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921886534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464796423912048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921902403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164618283510208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921918136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827652364969254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921934425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330695450305939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921934426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921934426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921958235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682861685752869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921958236, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921958236, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921974080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740956664085388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921989515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716862976551056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922005258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247275978326797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922012925, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922012926, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922036925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686084508895874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922036926, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922036926, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922044852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318586975336075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922060722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197687149047852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922076942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120435893535614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922092568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458795726299286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922092568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922092569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922116526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694817185401917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922116527, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922116527, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922132611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648127645254135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922148310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648699849843979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922163606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718554258346558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922171549, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922171550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922194568, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698918581008911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922194569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922194569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922202506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743145644664764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922217847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977541863918304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922233643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758914053440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922249562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467082262039185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922249563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922249563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922273194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770348310470581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922273194, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922273195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922288576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784374177455902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922304967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868702828884125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922320995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482455790042877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922328967, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922328967, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922351972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705785036087036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922351972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922351972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922359745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937333106994629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922375684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798265039920807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922391419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162185668945312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922407624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797341167926788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922407625, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922407625, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922431158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710275053977966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922431159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922431159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922447372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532531142234802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922463044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217393934726715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922479407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852251946926117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922487466, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922487466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922511187, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711264491081238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922511188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922511188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922518899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045834869146347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922534240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962071478366852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922550022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424989998340607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922565595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095310747623444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922565595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922565596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922588765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716414332389832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922588766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922588766, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922604546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064886748790741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922620694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089085042476654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922636527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082424521446228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922644308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922644309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922668061, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717027068138123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922668061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922668062, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922676900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023920029401779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922693126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484254151582718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922709257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245729774236679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922725401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330684274435043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922725402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922725402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922748761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717416882514954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922748762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922748762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922764628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101658046245575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922780989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392068862915039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922797257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078357696533203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922805269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922805269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922829705, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722246050834656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922829706, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922829706, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922837711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092657595872879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922854683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943704843521118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922870658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303018033504486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922886508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114925816655159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922886508, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922886509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922909796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726415395736694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922909797, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922909797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922925858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046390980482101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922942252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298200488090515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922958641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685152351856232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922966783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922966784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922990622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726582884788513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922990623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922990623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922998528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480943858623505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923015053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787256807088852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923030947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150174051523209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923046617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727818310260773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923046617, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923046618, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923069605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772840678691864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923069606, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923069606, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923085414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539046674966812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923100951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660900175571442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923117081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558316111564636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923125172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923125173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923148507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731557488441467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923148508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923148508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923156551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858536511659622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923172266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326175391674042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923188711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502087324857712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923204582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075931042432785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923204583, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923204583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923227787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734609842300415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923227788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923227788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923243143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066185146570206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923259008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379580408334732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923274652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350850224494934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923282421, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923282422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923305199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736172676086426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923305200, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923305200, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923313146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430043935775757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923329400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306946188211441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923345130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114463210105896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923361524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10337746143341064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923361525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923361525, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923385171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737131118774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923385172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923385172, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923401100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762783139944077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923416809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760640352964401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923432486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177191346883774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923440384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923440384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923463979, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738653421401978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923463979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923463980, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923471770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107596218585968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923487535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081405937671661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923503609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654085874557495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923519853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296954959630966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923519853, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923519854, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923542992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742135524749756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923542993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923542993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923558562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078573539853096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923575175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387252271175385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed595408662.log b/recommendation_v4/rcp_logs/gbs_32768/seed595408662.log new file mode 100644 index 000000000..7b0073954 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed595408662.log @@ -0,0 +1,641 @@ +:::MLLOG {"namespace": "", "time_ms": 1784923823966, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784923823966, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784923852552, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784923852555, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784923852555, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784923852555, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "seed", "value": 595408662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784923852556, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784923852610, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784923852611, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784924774858, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784924774859, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784924775192, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784924948773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836678862571716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924965581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383856236934662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924981566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858231902122498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924989482, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924989483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784925013821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500414252281189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784925013821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784925013821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784925021697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837924599647522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925036557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383790671825409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925051930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384791135787964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925067015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383552998304367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925067015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925067016, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925090444, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005645751953125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925090445, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925090445, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925105496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796211779117584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925120489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814091682434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925135875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802935183048248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925143241, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925143241, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925167377, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007614493370056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925167377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925167377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925175382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828180730342865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925190514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812199234962463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925206027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821853697299957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925221352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781917095184326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925221353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925221353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925247851, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009873509407043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925247851, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925247852, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925263086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814330101013184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925278386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374904066324234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925297149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781976699829102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925304640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925304641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925330423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012518763542175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925330423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925330423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925337933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370255947113037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925353590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720890879631042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925368733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686257600784302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925384277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694484531879425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925384278, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925384278, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925411601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015411376953125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925411601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925411602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925426679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370881199836731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925446704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13706547021865845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925462799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625629246234894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925470683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925470683, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925495725, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018681287765503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925495726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925495726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925503589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717231154441833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925519889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622015714645386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925535547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658837974071503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925551758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698041439056396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925551759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925551760, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925577871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022623538970947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925577872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925577872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925593559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361859142780304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925609566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349833756685257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925625364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668978214263916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925633055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925633056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925656880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026387572288513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925656881, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925656881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925665214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316628336906433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925683266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465803861618042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925698826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350046694278717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925714382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352211982011795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925714454, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925714455, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925742571, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030246376991272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925742572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925742572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925758028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388320803642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925773547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346368044614792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925789064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521713018417358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925797519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925797520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925825159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034403800964355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925825160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925825160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925833063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576149940490723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925848944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490794599056244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925864921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132160484790802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925881414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345817744731903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925881414, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925881415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925908207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039543509483337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925908207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925908207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925924389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391070067882538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925940477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352497935295105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925957102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489317893981934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925965427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925965428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925991891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048646926879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925991892, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925991893, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925999807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636374473571777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926015948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354897499084473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926032637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435180485248566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926048605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321246474981308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926048679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926048679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926074317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5057858228683472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926074318, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926074318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784926090081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293634355068207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926106798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128437101840973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926123070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330903172492981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926131018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926131018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926157606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5067757368087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926157607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926157607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926165533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841784000396729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926181419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113293051719666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926197864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992313504219055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926213905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045911490917206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926213906, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926213906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926240722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507960319519043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926240722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926240723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926256483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281035423278809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926272291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299067735671997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926289133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069604337215424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926297296, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926297297, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926323957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5093139410018921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926323957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926323958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926332013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169217109680176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926347868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508354127407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926363999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315698504447937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926380628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564418256282806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926380629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926380629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926406764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107851624488831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926406765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926406765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926423017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530486822128296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926438917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325453221797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926454815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323268294334412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926462762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926462762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926486098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122156143188477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926486098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926486099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926494950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639840483665466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926511856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528470695018768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926528422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362430453300476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926545030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364486813545227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926545030, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926545030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926569620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5136656165122986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926569621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926569621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926586392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486212491989136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926603371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308765411376953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926619410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623124361038208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926627737, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926627737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926652818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5151557922363281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926652818, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926652819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926660775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591039180755615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926677465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258016109466553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926693870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252952694892883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926710151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077950477600098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926710151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926710152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926736824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5165654420852661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926736825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926736825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926752965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108789920806885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926769347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819342851638794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926785760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296387016773224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926793851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926793851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926819857, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182176828384399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926819857, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926819858, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926827832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944668531417847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926844426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347389966249466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926860577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460499048233032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926877459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304914504289627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926877460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926877460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926904013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5202037692070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926904014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926904014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926920454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12826383113861084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926936607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192737102508545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926953000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562089204788208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926961144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926961145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926985666, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5223100781440735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926985667, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926985667, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926993803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665977120399475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927010504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360287249088287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927027333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13207198679447174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927043612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310240477323532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927043612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927043613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927070538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5247458219528198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927070539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927070539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784927086956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958255410194397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927103244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245639204978943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927121579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541290163993835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927129969, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927129970, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927156923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5276268720626831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927156924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927156924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784927165260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188722729682922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927181975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334594488143921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927198493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510122895240784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927214850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13468529284000397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927214851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927214851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927240358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5308718681335449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927240359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927240359, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927256765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602852821350098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927274324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398832082748413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927291228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331438422203064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927299709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927299710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927326780, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5344003438949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927326781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927326781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927335475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14049780368804932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927352245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521912693977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927369157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446122407913208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927386448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522741198539734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927386449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927386449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927413211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5380398631095886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927413211, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927413211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927429971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533861935138702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927446951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333868980407715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927463458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237205147743225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927472307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927472308, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927499324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5421988368034363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927499325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927499325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927507687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251972198486328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927524524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308952271938324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927540966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599780201911926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927557630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323758214712143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927557630, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927557631, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927582202, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5473196506500244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927582203, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927582203, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927598855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462544977664948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927616186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030727207660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927632926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187244534492493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927641274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927641275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927667758, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5539206266403198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927667758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927667758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927676144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235612213611603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927692704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075610995292664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927709473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331050843000412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927726631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328439563512802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927726632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927726632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927753949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5627309083938599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927753949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927753950, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927770534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196346163749695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927787068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12833121418952942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927803632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418219983577728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927811877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927811877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927838140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5725064277648926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927838140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927838141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927846570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494664430618286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927862957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12690362334251404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927884063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712113559246063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927901031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315246999263763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927901032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927901033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927926902, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5841225385665894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927926902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927926902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927943419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294320821762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927960072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891286611557007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927977241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281222701072693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927985244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927985244, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928010476, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5945836901664734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928010476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928010476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784928018800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302192062139511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928035906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12781557440757751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928052754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13022108376026154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928069264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870031595230103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928069265, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928069265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928095751, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6063676476478577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928095751, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928095751, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784928112326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270788460969925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928129367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009892404079437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928145799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331377506256104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928154060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928154061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928178286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6173183917999268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928178286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928178286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784928187129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317720115184784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928204417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12695279717445374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928221642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147717714309692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928238845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12690041959285736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928238846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928238846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928265610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6277042627334595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928265610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928265611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784928282830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935173511505127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928299543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035161793231964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928316641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12832790613174438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928325434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928325435, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928351644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6385121941566467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928351645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928351645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928360282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12764140963554382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928377167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777121365070343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928394324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847357988357544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928411108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736982107162476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928411109, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928411109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928437194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6502321362495422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928437195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928437195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928454293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12175190448760986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928471350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1209639385342598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928487891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934148520231247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928496416, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928496417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928522485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6597985029220581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928522486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928522486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928530758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114790827035904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928547546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11810548603534698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928564275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12251409888267517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928580585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966823786497116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928580586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928580586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928604821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.679889440536499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928604821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928604821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928622108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12079611420631409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928638965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219915747642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928655886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792543530464172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928664160, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928664161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928691170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6984253525733948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928691170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928691170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928699353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854451894760132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928716564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841905117034912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928733331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319189518690109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928749698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693151295185089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928749767, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928749768, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928777787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.714821457862854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928777788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928777788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928794439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549095809459686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928811035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547963321208954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928828050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885325819253922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928836254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928836255, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928862484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7276089787483215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928862485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928862485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928870730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238370090723038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928887946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11412002891302109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928904647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112364761531353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928921285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443521082401276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928921286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928921286, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928948806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7318027019500732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928948806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928948806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928965713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352169513702393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928982326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253584176301956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928999379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623548716306686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929007510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929007510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929035160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7387058734893799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929035160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929035160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784929043546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595771670341492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929060567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403662711381912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929077297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755603015422821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929093919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216030269861221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929093920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929093920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929121537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7406481504440308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929121538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929121538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784929138451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508982628583908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929155074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191394925117493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929171999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068151891231537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929179883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929179883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929205836, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7440747618675232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929205837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929205837, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784929214399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080348491668701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929231286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234578490257263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929248565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279229819774628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929264614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647105425596237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929264614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929264615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929289220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7441914677619934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929289220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929289221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784929307340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524942517280579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929324880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11904559284448624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929342297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695726960897446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929350979, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929350980, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929377777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7444848418235779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929377777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929377778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784929386229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151493564248085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929403834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11530593782663345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929420750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246289312839508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929437475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369481682777405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929437476, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929437477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929464601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7453886270523071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929464601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929464601, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929481810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815905570983887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929498933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709735542535782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929516551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120513826608658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929525059, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929525059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929552552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7482175230979919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929552552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929552553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929561136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552959680557251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929578522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557095497846603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929595382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111430823802948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929614103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430765151977539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929614104, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929614104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929640554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7511080503463745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929640554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929640554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929657331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667753219604492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929674544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543129593133926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929691425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606537759304047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929699985, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929699986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929727193, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7507250308990479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929727193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929727194, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929735750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394252091646194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929753074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155785620212555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929770017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755014419555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929787270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337989568710327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929787271, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929787271, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929815178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7519996762275696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929815179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929815179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929832403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704624652862549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929849114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643665492534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929866268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208968073129654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929874373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929874374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929901089, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7519577741622925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929901089, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929901090, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929909761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028270274400711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929926782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181627422571182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929944300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211200058460236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929960698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398860275745392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929960699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929960699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929985102, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7544057369232178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929985103, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929985103, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784930002446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638546943664551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930020041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623624920845032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930036877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679753124713898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930045663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930045664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930071822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7568017244338989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930071822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930071822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784930080293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073092669248581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930097492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996108129620552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930114666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795137286186218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930132071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104324571788311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930132072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930132072, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930156624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7576603889465332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930156624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930156625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784930173207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774759203195572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930191095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858386754989624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930208526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420484095811844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930217029, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930217030, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930244178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7562484741210938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930244179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930244179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784930252594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911398380994797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930269826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084250956773758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930286971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192867159843445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930304210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804858058691025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930304210, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930304211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930331038, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7573281526565552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930331039, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930331039, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784930348389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581943392753601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930364974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206581652164459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930382601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840746760368347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930391490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930391491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930417895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7596796154975891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930417895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930417896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784930426455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048819050192833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930443317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09950181841850281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930460427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146320179104805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930477681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132492125034332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930477681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930477682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930504688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592162489891052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930504689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930504689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930521743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622525215148926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930539269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866959393024445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930556302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830457508563995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930564539, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930564539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930589210, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7590803503990173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930589210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930589211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930598309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057723313570023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930616184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478857696056366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930633830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242997646331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930651570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310674250125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930651571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930651571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930678254, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.759053647518158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930678255, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930678255, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930695786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011617630720139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930713628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415384709835052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930731412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109878122806549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930739923, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930739924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930764997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7590778470039368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930764997, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930764997, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930773486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090514063835144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930791683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958805680274963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930809308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112897127866745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930826782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132475733757019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930826783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930826783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930853179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7617993950843811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930853180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930853180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930870660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037082225084305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930888688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312908679246902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930906142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708196997642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930914951, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930914952, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930940199, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628907561302185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930940199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930940199, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930948542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495848000049591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930966559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766910761594772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930983686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162195563316345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931001262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715072602033615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931001263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931001263, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931027788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634513974189758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931027788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931027788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784931044838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494755953550339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931062032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684455931186676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931079628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541891306638718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931088436, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931088437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931115308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7635741829872131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931115308, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931115309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784931123798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853531956672668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931140772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330960899591446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed651579272.log b/recommendation_v4/rcp_logs/gbs_32768/seed651579272.log new file mode 100644 index 000000000..db305a3b2 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed651579272.log @@ -0,0 +1,691 @@ +:::MLLOG {"namespace": "", "time_ms": 1784886606980, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784886606980, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784886631051, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "seed", "value": 651579272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784886631052, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784886631104, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784886631105, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784887531522, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784887531523, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784887531873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784887711877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906842470169067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784887727487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387837529182434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784887742186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899624347686768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784887749558, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887749559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887773835, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016107559204102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887773835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887773835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784887781419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899101316928864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887795850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896268606185913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887811100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886314630508423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887826220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880038261413574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887826221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887826221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887849569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017505884170532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887849569, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887849569, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784887864291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885259628295898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887879402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387483924627304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887894676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386648416519165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887901905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887901905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887925179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019131898880005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887925179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887925180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784887932850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872335851192474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887947855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385679394006729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887962999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385728120803833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887977938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835546374320984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784887977938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784887977939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784888000835, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021071434020996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784888000835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784888000835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784888015493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384839415550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888030255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138114333152771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888045122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382291316986084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888052512, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888052512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888075752, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023513436317444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888075753, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888075753, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784888082820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764053583145142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888098056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761968910694122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888112543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739246129989624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888127451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731086254119873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888127452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888127452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888151388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026282072067261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888151388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888151388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784888165755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745886087417603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888180449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741086423397064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888196065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675087690353394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888203601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888203601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888226973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029541850090027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888226973, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888226973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784888234538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373930275440216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888250576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651636242866516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888265609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684141635894775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888281735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722771406173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888281736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888281736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888304742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033634305000305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888304743, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888304743, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784888322785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644744455814362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888338622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522303104400635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784888353496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678282499313354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888360877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888360878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888384368, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503792941570282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888384368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888384368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784888391835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359138369560242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888407262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493895530700684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888422048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518017530441284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888436906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545306026935577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888436973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888436974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888461383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042440891265869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888461384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888461384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784888476223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340002566576004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888491433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347191482782364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888508532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527646660804749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888516792, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888516793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888540681, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047375559806824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888540681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888540681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784888548068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565899431705475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888563006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494490087032318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888578151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322707235813141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888593974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13459856808185577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888593974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888593975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888616819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053887963294983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888616820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888616820, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784888631966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376331329345703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888649264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529177010059357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888665356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349320113658905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888673137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888673138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888696069, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064141154289246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888696070, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888696070, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784888703592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644276559352875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888720478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348844647407532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888736266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13427196443080902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888751498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13207237422466278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888751570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888751571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888774670, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074495673179626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888774671, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888774671, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784888789531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13291579484939575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888805445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108178973197937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888820712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314732909202576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888828343, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888828344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888851075, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5085789561271667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888851075, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888851076, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784888858597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846680521965027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888873780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310528814792633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888889513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986384332180023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888904548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039179146289825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888904548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888904549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888927262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5098863840103149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888927263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888927263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784888942308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277080655097961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888957560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297703981399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888973476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060742616653442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784888980977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784888980977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784889003804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112761855125427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784889003805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784889003805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784889011369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13155606389045715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889026300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351446956396103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889041736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310132920742035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889057758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559769093990326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889057759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889057759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889080529, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5128117203712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889080529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889080530, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784889096017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524091243743896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889111456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326037496328354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889126883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318994641304016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889134521, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889134521, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889156994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5143459439277649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889156995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889156995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784889165855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641202449798584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889181725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511522114276886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889197420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627469539642334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889213167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637815415859222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889213167, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889213168, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889236248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5158920884132385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889236249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889236249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784889252853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347636580467224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889268821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093572854995728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889284587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362290382385254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889292626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889292626, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889315982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5173915028572083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889315983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889315983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784889323803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358761489391327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889339739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248589634895325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889355034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326661854982376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889370130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088509440422058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889370131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889370131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889392837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5188457369804382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889392837, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889392838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784889408190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309865117073059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889424088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837938010692596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889439611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951168417930603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889447309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889447309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889470252, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.520500898361206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889470252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889470252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784889477767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12927600741386414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889493282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476714491844177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889508776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345004141330719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889525285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303626298904419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889525286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889525287, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889547974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5224828720092773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889547974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889547975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784889563285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12801901996135712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889578624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13180002570152283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889594131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556404411792755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889602106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889602107, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889625691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.524665117263794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889625691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889625692, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784889633503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657613098621368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889649481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335998773574829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889665171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321249008178711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889680685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089843094348907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889680685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889680686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889703338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.527126669883728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889703339, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889703339, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784889719124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296313852071762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889734737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248883187770844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889750748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550090789794922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889758573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889758573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889781682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5300031900405884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889781683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889781683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784889789282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199348747730255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889804919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334475874900818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889820453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492831587791443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889835980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346670687198639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889835981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889835981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889860129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.533117413520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889860129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889860129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784889883148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605768978595734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889899934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338774561882019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889916001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315415382385254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889923959, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889923960, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889947220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5365585684776306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889947220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889947220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784889955371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14028900861740112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889971522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534507155418396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784889987805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442233204841614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890003943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523830473423004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890003944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890003944, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890027162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5400183200836182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890027162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890027162, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784890042811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519418239593506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890058556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331949681043625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890076683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231433928012848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890085371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890085372, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890109234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5441867113113403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890109235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890109235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784890117227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254261016845703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890132826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13076838850975037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890148344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583028316497803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890164139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322653442621231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890164139, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890164139, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890187570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.549422025680542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890187570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890187570, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784890203323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473527133464813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890219492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302829086780548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890235049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179311156272888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890242703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890242704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890266457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5562306642532349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890266458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890266458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784890274031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323864758014679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890289600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306699514389038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890305290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308139145374298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890321493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261987268924713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890321494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890321495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890345542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5654029250144958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890345543, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890345543, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784890361234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199448585510254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890376806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12822920083999634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890392121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134017676115036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890400004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890400007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890422715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5757771134376526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890422716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890422716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784890430601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479064404964447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890446248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12676578760147095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890462402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268426924943924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890478188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287129998207092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890478189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890478189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890501625, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5880189538002014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890501626, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890501626, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784890517179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929780781269073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890532695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285778284072876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890548739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12798969447612762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890556455, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890556456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890580478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5997128486633301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890580479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890580479, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784890588408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004116714000702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890608733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12749849259853363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890624258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979714572429657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890639751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12810684740543365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890639751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890639752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890663590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6127049326896667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890663590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890663590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784890678873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12654559314250946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890694834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948423624038696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890710538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256146013736725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890718316, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890718316, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890743096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6249397993087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890743097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890743097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784890751585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089877367019653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890767871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607181072235107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890784030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064022362232208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890799871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259201467037201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890799871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890799872, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890824532, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6376864910125732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890824533, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890824533, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784890840712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12842094898223877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890856736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12938256561756134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890873320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12749700248241425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890881333, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890881334, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890905554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6515070199966431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890905554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890905554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784890913410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649285793304443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890929175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12652578949928284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890945225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12676461040973663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890961214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259365975856781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784890961215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784890961215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784890986507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6658112406730652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784890986508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784890986508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784891002798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957870423793793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891018948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934898793697357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891034770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721716821193695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891042678, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891042679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891067085, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.677969753742218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891067086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891067086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784891074878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11937661468982697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891090884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615047603845596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891107049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12081261724233627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891122905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782926321029663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891122905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891122906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891147706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6982152462005615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891147706, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891147707, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784891164411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966632306575775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891180145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082813143730164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891195856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664722859859467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891203545, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891203546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891227701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7150721549987793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891227702, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891227702, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784891235445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11743553727865219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891251435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173960492014885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891267299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230888962745667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891282873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160268634557724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891282950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891282950, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891308136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7298416495323181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891308137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891308137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784891324330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463218182325363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891340333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456344276666641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891356622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838775336742401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891364654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891364654, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891388600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7397327423095703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891388600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891388600, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784891396700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172173917293549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891413118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274328082799911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891429395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041340976953506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891445920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327363550662994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891445921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891445921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891471063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7469193339347839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891471064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891471064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784891486784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160502582788467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891502429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107081919908524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891518239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535638570785522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891526073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891526073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891549608, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7512174248695374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891549608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891549608, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784891557323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448231548070908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891573372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308379471302032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891589119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662493854761124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891605269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093631386756897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891605269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891605270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891629146, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7536928653717041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891629147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891629147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784891645275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141892522573471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891661296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083892732858658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891677103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999979078769684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891684835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891684836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891708448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560389637947083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891708449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891708449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784891716346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999810695648193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891732410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11194638907909393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891748733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230692267417908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891764714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596001893281937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891764714, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891764715, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891789041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7566849589347839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891789042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891789042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784891806266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480924487113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891822503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906598508358002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891838909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11715337634086609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891846998, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891846998, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891870979, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.758203387260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891870980, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891870980, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784891879047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422158777713776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891895671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477014422416687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891911927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243162304162979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891927907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136232316493988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891927907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891927908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891952795, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597623467445374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891952795, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891952796, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784891969130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814562231302261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784891985110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11735286563634872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892001359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099065840244293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892009489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892009489, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892033171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606346011161804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892033171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892033171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784892041387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649218946695328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892057791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589295417070389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892073996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125297099351883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892090987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478861629962921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892090987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892090988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892113944, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7632928490638733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892113945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892113945, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784892129717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070249080657959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892145823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598494112491608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892161696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613615810871124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892169793, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892169793, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892193710, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7650867104530334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892193710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892193710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784892201778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045650765299797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892218211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180512607097626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892234370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079343929886818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892250985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127995103597641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892250986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892250986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892274233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651737928390503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892274233, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892274234, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784892290412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721711814403534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892306325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612587630748749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892322429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237470269203186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892330187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892330187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892354178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7665369510650635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892354178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892354178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784892362213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256901383399963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892378351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017247810959816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892394807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199380457401276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892410646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435178130865097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892410646, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892410646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892434178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683365345001221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892434179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892434179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784892450657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653980821371078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892466941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069025844335556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892482722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698974132537842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892490921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892490922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892514622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769115149974823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892514623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892514623, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784892522749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073540672659874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892538592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995936170220375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892554825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796601325273514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892571079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403429716825485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892571079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892571079, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892594100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695165872573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892594101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892594101, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784892609910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766206681728363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892626797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793063044548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892643102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429377108812332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892651073, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892651074, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892673966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683959007263184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892673967, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892673967, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784892681837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086614802479744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892697998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765066742897034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892713821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175756901502609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892729960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792073607444763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892729961, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892729961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892753087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708179354667664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892753088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892753089, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784892769384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536094009876251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892785292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192438215017319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892802129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861892998218536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892810462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892810463, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892834086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714892029762268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892834087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892834087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784892842171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465173423290253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892857996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995938628911972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892874089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402788758277893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892890055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108420044183731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892890055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892890056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892913767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714943885803223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892913768, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892913768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784892929815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645310580730438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892946244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845470428466797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892962384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845200717449188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784892970297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784892970298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784892993940, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711619138717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784892993941, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784892993941, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784893002536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049939692020416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893019198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482617259025574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893035648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245420575141907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893052128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297064274549484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893052128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893052129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893076328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713772654533386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893076328, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893076329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784893092693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022461950778961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893109300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377108097076416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893125900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078467220067978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893134110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893134110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893157688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717871069908142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893157689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893157689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784893165748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045512557029724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893182893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924027115106583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893199406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112511545419693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893215981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101587861776352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893215981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893215982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893239854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724250555038452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893239855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893239855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784893256393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992748290300369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893273278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269301176071167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893289925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11683948338031769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893298175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893298176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893322055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728994488716125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893322056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893322056, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784893330097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467509716749191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893347161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796239972114563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893363216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151535272598267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893379120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690972208976746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893379120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893379120, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893402342, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730903625488281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893402343, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893402343, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784893418451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465559363365173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893434429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664968192577362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893450788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542459785938263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893459095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893459096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893481535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732721567153931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893481536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893481536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784893489662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085653230547905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893505545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306200385093689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893522503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460115224123001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893538907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728271305561066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893538908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893538908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893561171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736385464668274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893561172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893561172, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784893577125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069732367992401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893593420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396308451890945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893609618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367564857006073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893617738, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893617738, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893641159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773847222328186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893641160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893641160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784893649367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411949455738068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893666257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317803919315338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893682663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369638890028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893699589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031976044178009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893699590, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893699590, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893722590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741410732269287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893722591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893722591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784893738838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070108413696289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893754970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080286055803299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893771084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163038223981857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893779080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893779080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893802190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735962271690369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893802191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893802191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784893810209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135368257761002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893826434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048162937164307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893842931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599635541439056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893859542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323463380336761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893859542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893859543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893882778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744584083557129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893882778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893882779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784893898764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750461369752884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893916185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386943817138672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893932506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591638088226318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784893940489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893940490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893966765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747220396995544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893966766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893966766, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784893974794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346657782793045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed72228511.log b/recommendation_v4/rcp_logs/gbs_32768/seed72228511.log new file mode 100644 index 000000000..972c26a19 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed72228511.log @@ -0,0 +1,684 @@ +:::MLLOG {"namespace": "", "time_ms": 1784916345392, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784916345392, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784916368526, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784916368529, "event_type": "POINT_IN_TIME", "key": "seed", "value": 72228511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784916368530, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784916368530, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784916368530, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784916368530, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784916368566, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784916368566, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784917330598, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784917330598, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784917330921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784917476758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772447407245636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917491789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826638460159302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917505944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138070210814476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917513105, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917513106, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917556678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989660978317261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917556678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917556679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917563805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137813538312912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917577861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778658211231232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917592515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798849284648895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917607002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789497315883636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917607003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917607003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917629223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4991390109062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917629224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917629224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917643705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712993264198303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917658160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374618411064148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917672792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732630014419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917679847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917679848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917702530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4993532598018646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917702530, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917702530, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917709928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775362074375153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917724565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773253560066223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917739455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780082762241364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917754227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13713935017585754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917754227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917754228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917776712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4996105134487152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917776713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917776713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917791280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780613243579865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917805702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367093175649643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917820501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741976022720337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917827756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917827757, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917850474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49990981817245483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917850475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917850475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917857553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609376549720764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917872479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644516468048096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917886697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360141634941101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917901328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613948225975037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917901329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917901329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917925376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002381205558777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917925377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917925377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917939506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656243681907654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917954054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644900918006897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917969359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356075406074524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917976691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784917976691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784917999324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005657076835632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784917999325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784917999325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918006715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670508563518524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918022290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542068004608154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918037169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13607509434223175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918052809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670150935649872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918052810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918052810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918076149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010197758674622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918076150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918076150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918090988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572272658348083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918106509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404619693756104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918121333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622118532657623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918128684, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918128685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918151660, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501443088054657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918151661, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918151661, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918158955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195976614952087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918173897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389018177986145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918188383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442328572273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918202868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134687602519989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918202938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918202938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918226872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501886248588562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918226873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918226873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918241353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312333822250366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918255906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417838513851166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918270579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487321138381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918278633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918278634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918300697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023907423019409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918300698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918300698, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918307864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355273425579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918322300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460753858089447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918337111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13135652244091034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918352684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436147570610046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918352685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918352685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918374790, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031273365020752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918374790, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918374791, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918389672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359442353248596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918405004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13528510928153992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918420757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496118783950806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918428340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918428341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918451422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042551159858704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918451423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918451423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918458857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650265336036682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918473963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133332759141922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918489418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431353867053986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918504364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191646337509155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918504426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918504427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918527175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053201913833618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918527176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918527176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918541930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327648162841797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918557673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311759650707245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918572439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330914944410324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918579964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918579964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918602349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064115524291992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918602349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918602349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918609704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836437463760376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918624675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091865181922913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918639921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979471683502197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918654472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035453855991364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918654473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918654473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918677126, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5076655149459839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918677128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918677128, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918691845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279159367084503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918706789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981469929218292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918722352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064764440059662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918729710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918729710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918752533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090273022651672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918752534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918752534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918760063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158594071865082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918774854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513100147247314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918790209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307851552963257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918805734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560907542705536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918805735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918805735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918828956, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5104570388793945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918828957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918828957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918843816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531525433063507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918858579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325646936893463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918873595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322141766548157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918881054, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918881055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918904018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5119597315788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918904019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918904019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784918912366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642945885658264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918927862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524779677391052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918943282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627271354198456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918958987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638947904109955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918958988, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918958988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918982063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5134583115577698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918982063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918982064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784918998168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469909131526947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919013478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308538019657135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919028636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363488882780075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919036438, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919036438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919059590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5149929523468018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919059591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919059591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919067213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597942888736725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919082982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132534459233284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919098170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262024521827698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919113213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309601366519928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919113213, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919113214, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919135932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5164604187011719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919135932, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919135933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919151239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310616135597229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919166943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828691840171814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919181802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954318523406982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919189240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919189240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919211832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5181271433830261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919211832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919211832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919219120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12932106852531433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919233987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465552031993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919249066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467355072498322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919265491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043341040611267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919265492, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919265493, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919288129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5201603174209595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919288130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919288130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919303359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808364629745483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919318813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188719749450684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919334553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354047656059265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919342421, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919342421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919365243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5223453640937805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919365243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919365244, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919372967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664932548999786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919388846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352487981319427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919403813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320524513721466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919418710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098083436489105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919418710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919418711, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919441785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248721837997437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919441786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919441786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919456821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956860661506653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919471874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325283646583557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919487826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354128122329712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919495664, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919495665, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919518508, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5279187560081482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919518509, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919518509, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919526154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204112648963928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919541820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342513144016266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919557178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496290147304535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919572601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464869558811188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919572601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919572602, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919595894, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5313082337379456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919595895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919595895, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919611427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605374097824097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919627289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339590847492218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919642880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331298053264618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919650560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919650560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919673510, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5350656509399414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919673511, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919673511, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919681576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14033368229866028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919697465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530099391937256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919713609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134381502866745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919729484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352352350950241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919729485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919729485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919752960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5390918850898743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919752960, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919752960, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784919768398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521337509155273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919783841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314424455165863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919799271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234010338783264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919807549, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919807550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919830679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5435341000556946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919830679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919830679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784919838388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251668214797974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919853650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090069591999054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919868707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359204351902008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919884172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231590390205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919884173, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919884173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919907268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5489684343338013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919907268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919907269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784919922746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346425712108612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919938457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301657259464264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919953737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192710280418396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919961340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919961340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919984793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5560663342475891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919984794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919984794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784919992182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234712183475494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920007441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055795431137085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920023028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311515748500824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920039002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258416950702667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920039003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920039003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920062206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5655450820922852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920062206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920062207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920077557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318531483411789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920092917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280774474143982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920108075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383027911186218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920115992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920115993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920138320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5758188366889954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920138321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920138321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920146135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457328081130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920161628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12660306692123413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920177661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12676239013671875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920193337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279688358306885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920193338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920193338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920216248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5875216722488403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920216249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920216249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920231533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916843593120575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920246889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283700168132782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920262610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777233123779297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920270244, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920270245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920293064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5971158146858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920293065, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920293065, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920300855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987399101257324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920316709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747779488563538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920332061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991659343242645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920347329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281830370426178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920347329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920347330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920370726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6084883809089661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920370727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920370727, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920385732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265394687652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920401166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956662476062775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920416429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265728950500488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920424064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920424065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920447766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6188040971755981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920447766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920447767, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920456151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13095517456531525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920472321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12618733942508698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920488288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087955117225647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920504103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12614627182483673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920504104, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920504104, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920527530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6290419697761536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920527531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920527531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920543695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12853315472602844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920559521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974947690963745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920576008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750142812728882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920583941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920583941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920606886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6402062773704529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920606887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920606887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920614609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267077624797821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920629993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12682293355464935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920645732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12737533450126648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920661415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12629444897174835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920661416, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920661416, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920684717, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6512894034385681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920684718, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920684718, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784920701093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12047374993562698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920716833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978594213724136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920732188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819213628768921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920739861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920739862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920762836, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6586418151855469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920762836, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920762836, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784920770412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12010522931814194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920785939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11710557341575623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920801770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214291900396347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920817263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884412169456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920817263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920817264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920841357, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6771609783172607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920841358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920841358, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784920857541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201605275273323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920872761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113944798707962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920888239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11734864860773087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920895876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920895876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920918471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.697370171546936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920918472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920918472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784920926064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11801479756832123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920941743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11807899922132492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920957334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131637841463089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920972628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681164801120758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920972709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920972709, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920997764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7185215950012207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920997764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784920997765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921013951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565908789634705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921029604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153610423207283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921045367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894449055194855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921053159, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921053160, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921076292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7351769208908081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921076292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921076293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921084080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309948563575745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921099970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140439510345459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921115868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248067021369934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921131912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430442333221436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921131913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921131913, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921155453, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7477874755859375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921155453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921155454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921170975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131657138466835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921186262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210916936397552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921201679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165657564997673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921209267, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921209267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921232642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7576781511306763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921232643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921232643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921240204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571135580539703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921255921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346253007650375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921271496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686680674552917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921287515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070648580789566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921287515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921287516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921312263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7630941271781921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921312264, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921312264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921328343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411026120185852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921344238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047783493995667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921360068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946878790855408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921367814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921367815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921390422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7656477093696594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921390422, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921390423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921398342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968473553657532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921414386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102154850959778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921430689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187690496444702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921446300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105411097407341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921446300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921446301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921470592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.765610933303833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921470593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921470593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921487685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11434149742126465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921503738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860518902540207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921519728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523887515068054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921527627, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921527628, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921550967, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662562131881714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921550968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921550968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784921558854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470872163772583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921574958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142851859331131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921590952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175619810819626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921606704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299964040517807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921606704, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921606704, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921631047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669826745986938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921631048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921631048, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784921647194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11742375791072845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921663019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694329977035522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921679285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066903173923492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921687358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921687358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921710893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671718597412109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921710894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921710894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784921718918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492278426885605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921735171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154143288731575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921751254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092615127563477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921767687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386583209037781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921767687, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921767687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921791880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686858773231506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921791880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921791881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784921807168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063108891248703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921822786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560918599367142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921838214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058741882443428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921845901, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921845902, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921869590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689475417137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921869590, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921869590, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784921877332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374341160058975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921893181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123684048652649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921908943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735724121332169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921925070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124037355184555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921925070, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921925071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921948098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696838974952698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921948099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921948099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784921963870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664421319961548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921979309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608026385307312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921994962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134241729974747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922002551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922002552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922025648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77006596326828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922025649, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922025649, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922033563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025160402059555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922049359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120078921318054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922065470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140459775924683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922081055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363482683897018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922081056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922081056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922105242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706637382507324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922105243, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922105243, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922121288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060083732008934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922137130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581669211387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922152387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647604614496231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922160466, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922160466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922183848, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712202072143555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922183849, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922183850, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922191714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683467984199524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922207067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936770051717758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922222723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763772577047348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922238472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391654819250107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922238473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922238473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922261690, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771274745464325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922261691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922261691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922276986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760411620140076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922293081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803252458572388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922308922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041300967335701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922316719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922316720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922339186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711797952651978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922339187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922339187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922347000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863196849822998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922363027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730031877756119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922378795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169744282960892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922394984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815933346748352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922394984, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922394985, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922418283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718651294708252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922418284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922418284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922434371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496190190315247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922450061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101616270840168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922466533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081785187125206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922474571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922474571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922497411, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720791697502136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922497412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922497412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784922505204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424646735191345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922520805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09959331899881363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922536713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143416315317154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922552379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066892743110657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922552380, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922552380, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922575153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772216260433197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922575154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922575154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784922590843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598835349082947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922606933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084340512752533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922622829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810564458370209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922630629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922630629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922654062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722917795181274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922654063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922654063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784922662836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954524576663971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922679249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145743802189827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922695633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200740933418274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922712017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263970285654068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922712018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922712018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922735401, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720116376876831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922735402, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922735402, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784922751541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977865755558014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922768260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354490369558334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922784831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046762764453888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922793001, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922793002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922816168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723965644836426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922816168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922816168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784922824232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040705442428589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922841268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952471941709518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922857450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124858558177948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922873660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108100414276123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922873661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922873661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922896654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727031707763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922896654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922896654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784922912937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993266850709915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922929404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11295507103204727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922945794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169310137629509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922953891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922953892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922978128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727841138839722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922978129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922978129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784922986012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471153050661087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923002883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759150981903076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923018915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122618079185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923034864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067088171839714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923034865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923034865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923058955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729902863502502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923058956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923058956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784923075046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469332337379456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923091001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629673302173615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923107361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516456514596939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923115577, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923115577, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923139323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731150388717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923139324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923139324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784923147351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846081376075745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923163159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033557653427124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923179803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478110611438751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923195772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726742446422577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923195773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923195773, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923218929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732696533203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923218930, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923218930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784923234429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038822889328003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923250541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386776924133301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923266438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296221822500229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923274390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923274390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923298171, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734071016311646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923298172, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923298172, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784923306160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381785035133362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923322565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030702069401741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923338400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330454796552658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923354830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312199592590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923354831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923354831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923378787, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734457850456238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923378788, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923378788, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784923394912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719934105873108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923411033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701557993888855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923427171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145726054906845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923435189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923435189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923458785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734838128089905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923458786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923458786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784923466933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137020796537399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923483201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004648357629776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923499703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105995774269104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923516371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333159565925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923516371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923516372, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923540528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736527323722839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923540528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923540528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784923556463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731489211320877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923573340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396728664636612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed764588808.log b/recommendation_v4/rcp_logs/gbs_32768/seed764588808.log new file mode 100644 index 000000000..e011597ff --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed764588808.log @@ -0,0 +1,675 @@ +:::MLLOG {"namespace": "", "time_ms": 1784909144545, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784909144545, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784909167735, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784909167741, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "seed", "value": 764588808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784909167742, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784909167773, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784909167774, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784910117646, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784910117647, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784910117953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784910312979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820330798625946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910328476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382463276386261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910343211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836143910884857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910350642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910350643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910394067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034065842628479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910394067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910394067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910401281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823899626731873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910415729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827399909496307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910430695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836318254470825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910445523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829629123210907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910445523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910445523, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910467889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035640597343445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910467889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910467889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910482903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793504238128662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910497819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802890479564667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910512847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786938786506653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910520143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910520143, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910542900, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037497878074646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910542900, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910542900, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910550474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818907737731934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910565491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805946707725525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910580739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805735111236572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910595922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759109377861023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910595923, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910595923, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910618535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039834976196289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910618536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910618536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910633381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380254626274109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910648203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731259107589722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910663051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764618337154388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910670413, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910670414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910693253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042589902877808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910693254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910693254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910700427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674083352088928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910715605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695120811462402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910730017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664567470550537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910745068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366850882768631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910745069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910745070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910768113, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045554637908936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910768114, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910768114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910782627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692858815193176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910797576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685627281665802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910813197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13607747852802277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910820596, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910820597, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910843719, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048900842666626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910843719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910843719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910851158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369481086730957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910867064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359652578830719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910882190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634032011032104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910898092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368880271911621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910898093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910898093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910920394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053009390830994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910920394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910920394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910935279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586190342903137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910950820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455606997013092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910965845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13633683323860168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784910973424, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910973425, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910996471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5057012438774109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910996472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784910996472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911003976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325257271528244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911019335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426047563552856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911033847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469113409519196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911048457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499926030635834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911048537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911048538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911072212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060967803001404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911072213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911072213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911086940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345035910606384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911101736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342620998620987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911116625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350092887878418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911124699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911124700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911147932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5065435171127319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911147933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911147933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911155075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563615083694458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911169532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346539855003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911184406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13161532580852509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911199899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440628349781036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911199900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911199900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911221919, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071179866790771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911221920, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911221920, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911236919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367921113967896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911252188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515013456344604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911267912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489103317260742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911275530, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911275531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911297677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5080927014350891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911297677, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911297677, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911305096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639843463897705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911320213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339300453662872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911335738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423292338848114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911350650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201023638248444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911350712, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911350712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911373278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090498924255371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911373279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911373279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911388164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278095424175262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911403841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13120678067207336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911418819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313302397727966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911426402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911426402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911448535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100327134132385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911448535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911448535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911456000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850805163383484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911471255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310124546289444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911486798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981727719306946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911501607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303306519985199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911501608, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911501609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911524400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5112288594245911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911524401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911524401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911539309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276566565036774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911554481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297786682844162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911570384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064062595367432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911577921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911577921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911600441, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5124815702438354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911600441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911600442, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911608087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150522112846375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911623105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520218431949615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911638668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313645124435425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911654205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569998741149902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911654205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911654206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911676643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138489007949829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911676644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911676644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911691683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350744515657425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911706789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259848952293396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911722132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321472704410553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911729762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911729762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911753110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5151954889297485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911753111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911753111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911761856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636158406734467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911777690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526569306850433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911793264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362164318561554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911809148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647061586380005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911809149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911809149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911832499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5165829062461853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911832499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911832500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911848917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347959041595459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911864546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082635402679443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911880019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620224595069885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911887946, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911887946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911910777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5179732441902161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911910777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911910777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911918469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588255643844604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911934727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324286013841629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911949813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255886733531952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911964828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308419108390808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911964829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911964829, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911987266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5192909240722656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911987267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784911987267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912002570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111421465873718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912018601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828599452972412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912033951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959140539169312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912041570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912041570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912064263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5207582712173462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912064263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912064264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912071661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928323447704315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912086931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347729116678238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912102211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345614790916443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912118606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303723156452179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912118606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912118607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912141874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5225046873092651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912141875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912141875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912157031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813451886177063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912172407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319412887096405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912187763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353393793106079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912195559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912195560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912219454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5244052410125732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912219455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912219455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912227190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662438094615936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912242823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354535400867462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912257791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206999003887177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912272716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13095015287399292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912272717, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912272718, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912295621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.526627242565155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912295622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912295622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912310873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954393029212952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912326193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255901634693146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912341872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353662610054016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912349692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912349692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912372448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5293434262275696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912372449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912372449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912380097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131979301571846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912395705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333524763584137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912411199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508671522140503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912426804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346048265695572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912426804, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912426805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450580, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5323871374130249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450581, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912450581, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912466256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603250682353973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912482474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398496806621552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912498114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331184357404709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912505771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912505772, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912528338, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5358318090438843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912528338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912528339, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912536294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14031833410263062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912552246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520775735378265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912568497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450711965560913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912584719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525578379631042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912584719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912584720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912608445, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5395123362541199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912608446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912608446, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912623866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524042069911957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912639409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322031497955322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912655062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225343823432922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912663525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912663526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912686861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5439045429229736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912686862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912686862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912694852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241519033908844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912710621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13074944913387299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912726422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580766320228577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912742414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231085240840912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912742415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912742415, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912765978, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5495175719261169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912765979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912765979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912781861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134669691324234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912797982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013997673988342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912813179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187026977539062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912820767, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912820767, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912843655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.557129442691803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912843655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912843656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912851093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227570056915283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912866314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063809275627136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912881853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298597931861877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912897685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249635696411133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912897686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912897686, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912921075, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5674247741699219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912921076, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912921076, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784912936911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193079829216003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912952752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12797290086746216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912968287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338280290365219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912976288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912976288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912999447, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5787985920906067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912999447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784912999448, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913007400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448476791381836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913023141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265079379081726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913039359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12664327025413513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913055042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252101838588715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913055043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913055044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913078373, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5906544327735901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913078373, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913078373, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913093462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880684435367584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913108488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813392281532288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913124006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274450570344925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913131589, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913131590, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913155043, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6010954976081848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913155044, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913155044, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913162866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295657455921173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913178716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12667112052440643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913194553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931089103221893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913210344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758558988571167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913210345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913210345, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913233896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6105836033821106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913233897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913233897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913249248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12587103247642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913265169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902747094631195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913280960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170839846134186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913288783, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913288784, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913312293, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6180275082588196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913312294, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913312294, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913320629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030223548412323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913336577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1253012716770172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913352293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013193011283875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913367960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12489432841539383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913367961, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913367961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913391288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6242179274559021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913391289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913391289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913407283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276513934135437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913423281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12887048721313477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913439851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12667803466320038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913448065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913448065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913471079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6298779845237732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913471079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913471079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913479103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12594611942768097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913494892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12597566843032837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913511032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584879994392395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913527098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546218931674957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913527099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913527099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913550413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6346467137336731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913550414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913550414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913566698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11883053928613663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913582485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11897464841604233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913597912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116519495844841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913605658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913605659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913628656, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6400547623634338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913628657, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913628657, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913636202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11873442679643631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913651777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557237058877945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913667590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12053964287042618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913683125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11756888031959534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913683126, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913683126, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913707262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6464189887046814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913707263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913707263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913723558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935362219810486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913739162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056698113679886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913754864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663470417261124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913762544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913762544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913785177, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.654391348361969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913785177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913785178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913792911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758242547512054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913808904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752405017614365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913824788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125873401761055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913840328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626480519771576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913840402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913840403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913864413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6627070307731628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913864415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913864415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784913880380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11534138023853302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913895938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506126075983047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913911716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811086744070053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913919479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913919479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913942763, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.672302782535553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913942764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913942764, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784913950606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250068992376328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913966556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438177525997162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913982533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181698739528656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913998582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480274051427841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913998582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784913998583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914022708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6823498010635376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914022709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914022709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914038306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358526349067688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914053687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322459578514099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914069283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11745114624500275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914076991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914076991, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914099892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6943596601486206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914099893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914099893, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914107448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700191557407379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914123303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552466452121735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914139051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929881036281586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914154980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374523490667343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914154980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914154981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914178892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7104621529579163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914178893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914178894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914195174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11716657876968384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914211122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440394818782806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914226973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11276461184024811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914234667, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914234667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914257521, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7282726168632507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914257522, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914257522, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914265529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341209709644318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914281646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527660489082336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914297848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471851170063019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914313426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981239378452301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914313426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914313427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914337248, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7435504198074341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914337249, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914337249, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914354208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178157702088356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914370202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215115562081337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914386481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11879980564117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914394513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914394513, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914416691, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7532715201377869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914416692, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914416692, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914424663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11839203536510468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914441197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.118648000061512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914457535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523827910423279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914473620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589697003364563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914473621, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914473622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914497261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7585479617118835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914497262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914497263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914513374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215931698679924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914529183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956530064344406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914545458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273716390132904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914553505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914553506, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914575986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7611832618713379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914575986, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914575987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914584105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815893650054932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914600473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727094650268555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914616570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346898972988129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914633372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669398307800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914633372, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914633373, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914655891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.763435423374176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914655892, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914655892, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914671582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874494910240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914687515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779502987861633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914703279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786817967891693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914711178, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914711178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914734314, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7645410895347595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914734315, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914734315, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914742213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562983900308609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914758504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220568418502808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914774687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912683606147766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914791205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402136087417603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914791206, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914791207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914813696, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7654991149902344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914813696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914813696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784914829742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830221325159073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914845456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738680511713028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914861426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274343937635422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914869100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914869100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914891986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661555409431458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914891987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914891987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784914899985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388831049203873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914915984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174589604139328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914932282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280366778373718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914948096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460726171731949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914948097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914948097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914971501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676534652709961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914971502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914971502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784914987757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698585957288742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915003896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066393107175827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915019501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747850686311722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915027700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915027701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915050060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7684279084205627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915050061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915050061, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915058082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703179240226746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915073733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10050616413354874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915089838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778223723173141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915106017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449586063623428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915106017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915106018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915129228, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692233920097351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915129229, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915129229, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915144997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752008855342865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915161878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838459432125092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915178077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487435758113861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915186013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915186013, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915209107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694152593612671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915209108, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915209108, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915216969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950776189565659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915233127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835957527160645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915248999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222866922616959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915265250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811607539653778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915265250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915265250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915288422, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701581716537476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915288423, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915288423, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915304648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567232221364975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915320365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209030658006668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915336668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856326669454575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915344899, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915344899, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915368381, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701243162155151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915368382, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915368382, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915376321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468561947345734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915392029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09985316544771194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915408039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466952413320541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915423887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091748625040054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915423888, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915423888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915447367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708522081375122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915447367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915447368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915463378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678555816411972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915479747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881776362657547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915495896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844854265451431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915503778, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915503779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915527643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771024763584137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915527644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915527644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915536466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013227701187134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915553062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444812268018723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915569451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287456750869751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915585789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302164196968079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915585790, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915585790, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915609190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712953686714172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915609191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915609191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915625231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011148989200592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915641736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393638700246811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915658117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105769872665405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915666230, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915666231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915690068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715229988098145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915690068, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915690069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784915698112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102274060249329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915714956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982854664325714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915731340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300218105316162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915747721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100876331329346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915747721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915747722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915770815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719113230705261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915770815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915770815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784915787287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104167178273201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915804211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11321032792329788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915821063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703985929489136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915829406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915829406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915853342, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720591425895691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915853342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915853343, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784915861337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476582288742065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915878217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779406875371933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915894485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157927870750427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915910637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717672109603882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915910638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915910638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915933585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719971537590027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915933586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915933587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784915949638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105145163834095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915965696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066320538520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915982151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492795705795288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915990418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784915990418, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916013957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722120881080627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916013958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916013958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784916022066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858382284641266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916037909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344737023115158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916054827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477666556835175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916071357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741794109344482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916071357, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916071357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916094943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724264860153198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916094943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916094944, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784916110714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054334998130798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916126925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039135605096817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916142896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358244925737381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916150904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916150905, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916173890, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725723385810852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916173891, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916173891, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784916181957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044023185968399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916198665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315615683794022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916214817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419813334941864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916231310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320920497179031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916231310, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916231311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916254463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729415893554688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916254464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916254464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784916270615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739464312791824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916286624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744218528270721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916302611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184705257415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916310536, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916310537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916334463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773004412651062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916334464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916334464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784916342420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142387449741364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916358513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057319700717926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed770053153.log b/recommendation_v4/rcp_logs/gbs_32768/seed770053153.log new file mode 100644 index 000000000..78782600f --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed770053153.log @@ -0,0 +1,634 @@ +:::MLLOG {"namespace": "", "time_ms": 1784909143635, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784909143635, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784909167457, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "seed", "value": 770053153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784909167459, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784909167460, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784909167460, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784909167460, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784909167460, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784910140408, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784910140409, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784910140735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784910324800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389344185590744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910340380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389683485031128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910355456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872721791267395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784910362949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910362950, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910405874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020070672035217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910405875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910405875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784910413330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877063989639282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910428002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387576162815094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910443327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874205946922302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910458663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860732316970825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910458664, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910458664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910484483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021510720252991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910484483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910484484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784910499523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386704444885254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910514811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855046033859253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910531115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385268270969391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910538575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910538575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910562561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023193359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910562562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910562562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784910570332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850875198841095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910585638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839450478553772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910600945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840480148792267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910616480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813842833042145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910616481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910616481, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910641708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025081634521484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910641709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910641709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784910659385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829465210437775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910674476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782820105552673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910689663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814805448055267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910697270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910697270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910721505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502726137638092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910721506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910721506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784910728851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373191773891449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910744704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374582052230835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910759463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714216649532318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910774857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712553679943085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910774858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910774859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910799992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029904842376709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910799992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910799993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784910814967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718166947364807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910830296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720062375068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910846231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655930757522583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910853925, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910853925, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910876912, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503267765045166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910876913, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910876913, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784910884635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137345552444458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910904931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631121814250946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910920451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365870237350464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910936676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371016949415207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910936676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910936677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910960894, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036372542381287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910960895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910960895, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784910976184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618811964988708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784910992273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135063037276268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784911007603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656127452850342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911015237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911015238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911038678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039888620376587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911038679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911038679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784911046292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315671682357788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911061941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13461041450500488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911077153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349869668483734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911094729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520447909832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911094805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911094806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911120492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5043696165084839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911120492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911120493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784911135560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338566094636917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911150704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345287263393402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911165935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350940465927124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911174227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911174228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911200188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048134922981262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911200189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911200189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784911207633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568013906478882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911222992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484889268875122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911238549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184264302253723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911254741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453221321105957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911254742, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911254742, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911280018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054461359977722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911280019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911280019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784911295831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365161418914795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911311539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526003062725067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911327729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498932123184204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911335692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911335692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911360717, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064824819564819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911360717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911360717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784911368475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653329014778137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911384268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338954746723175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911400966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423937559127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911416507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208194077014923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911416575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911416576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911441281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507524311542511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911441281, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911441282, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784911456690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287732005119324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911472985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111160695552826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911488636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329931914806366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911496393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911496394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911522118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086269378662109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911522118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911522118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784911529905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384134590625763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911545726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309875249862671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911561904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992608547210693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911577609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035142421722412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911577610, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911577610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911603005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5098665356636047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911603005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911603005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784911618463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278046250343323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911634123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297377496957779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911650161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058561086654663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911658120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911658121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911683114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.511155366897583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911683115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911683115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784911690831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131535142660141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911706357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505303859710693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911722163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331595629453659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911738426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565003871917725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911738427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911738427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911763911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125112533569336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911763912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911763912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784911779555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518072664737701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911795360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247103989124298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911811225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323965668678284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911819050, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911819051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911842550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138816237449646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911842551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911842551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784911851318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627293705940247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911867780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515758514404297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911883908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621678948402405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911900146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637763261795044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911900146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911900147, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911924292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.515277087688446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911924293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911924293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784911941049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347421407699585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911957420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078640401363373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911973437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13619214296340942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784911981696, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784911981696, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784912005788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5167177319526672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784912005789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784912005789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784912013729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590103387832642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912030143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251669704914093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912046031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326831579208374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912061698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082003593444824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912061699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912061699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912088139, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5180983543395996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912088140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912088140, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784912103799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310250163078308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912120075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818423449993134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912136126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294378936290741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912144112, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912144113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912169212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5196504592895508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912169213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912169213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784912176933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926052510738373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912192994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469800353050232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912208903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344725787639618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912225321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303904950618744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912225322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912225322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912251213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5216056704521179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912251214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912251214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784912267091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800781428813934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912282589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200470805168152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912298817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546030223369598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912306789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912306790, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912330604, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5237603187561035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912330605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912330605, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784912338663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668221235275269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912355053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352973759174347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912371026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321452260017395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912386701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085110485553741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912386702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912386702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912412376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5262863039970398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912412377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912412377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784912428409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952736020088196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912444273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245420157909393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912460639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538897037506104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912468644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912468645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912494404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5291929841041565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912494405, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912494405, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784912502338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198266923427582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912518175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333407163619995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912534027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349261999130249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912549991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458001613616943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912549992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912549992, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912573620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5327134728431702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912573621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912573621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784912589935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601604104042053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912606981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389244675636292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912623671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330435872077942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912631845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912631845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912658481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5368900299072266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912658481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912658481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784912667046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14015386998653412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912683508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516464829444885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912700191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434410095214844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912716877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511916995048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912716877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912716878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912743300, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5415626764297485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912743301, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912743301, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784912759847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351929008960724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912776476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314247131347656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912792906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221541047096252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912801754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912801755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912828014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5467609167098999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912828015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912828015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784912836415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323441118001938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912853015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13068026304244995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912869174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566631078720093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912885516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210713863372803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912885516, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912885517, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912909337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5530866384506226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912909338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912909338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784912925786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448208570480347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912942798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299229860305786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912959330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158106803894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784912967453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912967454, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912993652, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5618466138839722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912993652, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784912993652, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784913001856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184881210327148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913018182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13021688163280487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913034807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253261148929596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913051441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204127550125122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913051441, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913051442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913077882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5731747150421143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913077883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913077883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784913094148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13135573267936707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913110336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272072196006775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913126776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303112983703613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913134804, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913134805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913160450, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5840501189231873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913160451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913160451, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784913168717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352680206298828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913184754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255374252796173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913201297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12517598271369934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913218016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113750517368317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913218016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913218017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913243771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5993665456771851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913243771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913243771, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784913260024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274404525756836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913276419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659335136413574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913293367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12566183507442474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913301407, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913301408, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913325724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.609666645526886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913325725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913325725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784913334001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277349293231964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913350544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12455026060342789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913367193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745235860347748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913383641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12525179982185364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913383642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913383642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913409772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6282863020896912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913409773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913409773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784913426244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12292923033237457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913443110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12631632387638092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913459442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12848085165023804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913467681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913467681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913491727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6530503034591675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913491728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913491728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784913500595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12677103281021118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913517608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12203839421272278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913534624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126408189535141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913551644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063780426979065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913551644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913551645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913577929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6880220174789429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913577929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913577930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784913594922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12370394915342331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913611608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12477058172225952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913628813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12265840917825699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913637371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913637371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913663204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.732940137386322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913663205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913663205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784913671722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12131274491548538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913688469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162134051322937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913705678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12069230526685715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913722484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11985331773757935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913722485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913722485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913746928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7489625811576843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913746928, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913746929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784913763984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281489580869675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913780657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181402951478958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913797122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000147461891174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913805581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913805581, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913831877, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7544142603874207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913831877, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913831877, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784913840059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11165246367454529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913856539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867953300476074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913873192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484955251216888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913889534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228568851947784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913889534, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913889534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913913547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560238242149353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913913548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913913548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784913930726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401589214801788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913947243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390228778123856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913963673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089034378528595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784913971927, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913971928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913997933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7606884837150574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913997933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784913997934, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784914005924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353965103626251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914022928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204653233289719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914039575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081051304936409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914055764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138410866260529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914055845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914055845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914082491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7630901336669922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914082492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914082492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784914098767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912621021270752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914114992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027466505765915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914131388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346612334251404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914139331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914139331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914165198, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7654290199279785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914165199, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914165199, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784914173282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806182026863098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914190173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813065618276596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914206709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555949807167053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914223634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920141637325287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914223634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914223635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914249103, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678111791610718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914249104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914249104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784914265844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741252452135086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914282269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641105473041534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914299137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181633174419403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914307089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914307089, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914333151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7686997056007385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914333152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914333152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784914341481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143506526947021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914358429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073321849107742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914375123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047273725271225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914391965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783860087394714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914391965, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914391966, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914418501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769637405872345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914418502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914418502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784914435039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174342781305313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914451498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912062227725983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914468264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765811800956726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914476201, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914476201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914501495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702892422676086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914501496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914501496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784914509808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880566388368607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914526638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990476608276367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914543551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143650114536285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914559688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049978956580162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914559688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914559688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914584679, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7706796526908875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914584680, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914584680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784914602620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455186456441879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914619948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11868801712989807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914637473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536802351474762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914646160, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914646161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914672071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771597683429718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914672071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914672071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784914680433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498531699180603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914698129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425146460533142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914714957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117720901966095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914731512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374025046825409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914731513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914731513, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914758204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716794013977051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914758204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914758204, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784914775279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11744248867034912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914792164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11723889410495758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914809570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063133925199509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914818115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914818116, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914844412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718966603279114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914844413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914844413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784914852919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529085040092468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914870213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488215625286102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914887195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108186841011047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914905071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455073416233063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914905072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914905072, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914931032, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771872878074646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914931033, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914931033, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784914947948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066843718290329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914965005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609996318817139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914981883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606445372104645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784914990485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784914990485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784915016022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771892786026001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784915016022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784915016022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784915024588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430027544498444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915041875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117892861366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915058690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793137550354004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915075995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284301429986954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915075996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915075997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915103496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723210453987122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915103496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915103497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784915120423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732710361480713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915137025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700968652963638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915154279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021600067615509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915162448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915162449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915189185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725693583488464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915189186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915189186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784915197784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330033302307129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915214827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202473402023315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915232220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149334907531738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915248761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435442626476288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915248762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915248762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915273058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772398829460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915273059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915273059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784915290268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629518330097198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915307886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068354994058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915324795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070491373538971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915333675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915333677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915359542, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724577188491821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915359542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915359542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784915368029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785210877656937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915385216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036914050579071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915402479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839095711708069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915419637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504861176013947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915419637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915419638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915443658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727499604225159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915443658, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915443658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784915460240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845385491847992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915477701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887239873409271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915495185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488373041152954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915503683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915503684, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915530078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730438113212585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915530078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915530078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784915538423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973086208105087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915555529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785815119743347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915572761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215335130691528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915590095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843516141176224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915590096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915590096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915616229, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731454372406006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915616230, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915616230, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784915633541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588090121746063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915650164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237767547369003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915667809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821081697940826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915676605, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915676606, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915702742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731905579566956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915702742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915702743, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784915711262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10493486374616623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915727966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976956248283386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915745148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500424891710281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915762433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137603968381882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915762433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915762433, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915789066, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732917666435242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915789066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915789067, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784915806085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720402002334595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915823454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903333127498627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915840388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877501219511032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915848639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915848639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915872539, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735381722450256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915872540, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915872540, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784915881549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999657958745956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915899318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148197203874588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915916683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251301318407059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915934112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379208415746689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915934113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915934113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915960359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773662805557251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915960359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915960359, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784915977513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030814051628113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784915995054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394002288579941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916012800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086305230855942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916021285, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916021286, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916046014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738317251205444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916046014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916046015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784916054543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127790808677673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916072453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982535779476166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916090136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320339143276215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916107687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133429408073425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916107688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916107688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916133483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738375067710876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916133483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916133484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784916151079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036783456802368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916169128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322231590747833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916186564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698813736438751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916195255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916195256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916220650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738844156265259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916220650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916220651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784916228858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539086699485779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916246709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803443193435669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916263613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018192395567894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916280666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071152538061142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916280666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916280667, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916306749, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773594081401825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916306749, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916306749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784916323461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516075789928436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916340332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674521327018738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784916357740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582875460386276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed813474716.log b/recommendation_v4/rcp_logs/gbs_32768/seed813474716.log new file mode 100644 index 000000000..2b271ee70 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed813474716.log @@ -0,0 +1,649 @@ +:::MLLOG {"namespace": "", "time_ms": 1784865317254, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784865317254, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784865343759, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784865343760, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "seed", "value": 813474716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784865343761, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784865343784, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784865343784, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784866238014, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784866238015, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784866238325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784866386407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889388740062714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784866401786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889822363853455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784866416614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886550068855286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784866423996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866423997, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866473498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49992015957832336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866473499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866473499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784866480984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879427313804626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866495633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138699010014534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866510990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867506384849548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866525890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865134119987488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866525891, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866525891, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866549071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000882148742676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866549071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866549071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784866563921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854649662971497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866578861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847501575946808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866593981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382513791322708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866601229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866601230, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866624355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002747178077698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866624356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866624356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784866632131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832880556583405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866647189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838960230350494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866662420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838106393814087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866677681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806653022766113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866677681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866677682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866700811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005096793174744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866700811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866700812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784866715895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824012875556946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866730737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378125548362732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866745907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802699744701385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866753396, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866753397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866776776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007984042167664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866776777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866776777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784866784153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738197088241577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866799551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745559751987457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866814135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718438148498535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866829115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722258806228638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866829116, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866829116, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866853042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5011144876480103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866853043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866853043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784866867994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740065693855286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866883196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373768448829651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866898982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659091293811798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866906525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866906526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866929724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015040040016174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866929724, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866929725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784866937405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13738906383514404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866953481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652417063713074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866968715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13676974177360535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866984639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714928925037384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784866984640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784866984641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784867007700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019540786743164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784867007701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784867007701, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784867022974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364782154560089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784867038859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537099957466125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784867053901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13664430379867554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867061414, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867061414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867084825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024121403694153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867084825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867084826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784867092347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343486189842224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867107953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349656581878662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867122880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523045182228088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867137964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534927368164062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867138039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867138039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867162139, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502917468547821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867162139, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867162139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784867177222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404159247875214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867192253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476285338401794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867207426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537245988845825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867215645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867215645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867239454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034622550010681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867239455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867239455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784867247023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358211785554886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867262243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508576154708862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867277726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224801421165466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867293705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347406804561615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867293706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867293706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867316764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041109919548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867316764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867316765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784867332275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390648365020752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867347990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352284848690033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867364120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349969208240509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867372047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867372048, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867395060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5050241351127625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867395061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867395061, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784867402814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636213541030884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867418579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351479172706604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867434883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435077667236328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867450577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321287900209427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867450642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867450643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867474150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060195326805115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867474150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867474151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784867489613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294489681720734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867505856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125818967819214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867521415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330741047859192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867529209, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867529209, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867552109, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071291327476501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867552110, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867552110, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784867559784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837289810180664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867575371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311246007680893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867591459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998881936073303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867607109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047222793102264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867607110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867607110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867630289, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5083502531051636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867630290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867630290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784867645801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273750245571136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867661545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984631955623627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867677782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069316744804382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867685494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867685494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867708516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096793174743652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867708517, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867708517, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784867716263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159923255443573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867731620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515201210975647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867747439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317684829235077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867763486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356806457042694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867763487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867763488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867786531, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5109959840774536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867786532, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867786532, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784867802159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351565569639206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867817928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248607516288757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867833742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316947221755981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867841591, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867841592, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867865299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5122719407081604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867865299, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867865299, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784867873932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636627793312073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867889828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516679406166077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867905602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632236421108246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867921619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646163046360016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867921619, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867921620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867945271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5134730935096741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867945271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867945272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784867961900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13483776152133942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867978070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307704597711563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784867994047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362009346485138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868002202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868002203, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868026125, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5147641897201538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868026126, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868026126, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784868034059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592682778835297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868050071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252314925193787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868065764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13274820148944855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868081193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080215454101562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868081194, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868081194, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868104294, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.516205370426178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868104295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868104295, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784868119882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109523057937622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868135895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824208080768585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868151991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956905364990234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868159924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868159925, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868183154, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5177932381629944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868183154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868183154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784868190978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293424516916275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868206940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470570743083954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868222840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344418227672577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868239330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032689690589905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868239331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868239331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868262595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5194911360740662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868262595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868262595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784868278120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800902128219604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868293666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198813796043396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868309290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569359481334686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868317256, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868317256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868341263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5213505029678345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868341263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868341264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784868349138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653047382831573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868365340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355834782123566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868381385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200271129608154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868397189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098497688770294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868397190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868397190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868420903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5234540700912476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868420904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868420904, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784868436925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961238622665405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868452826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324542760848999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868469012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354411542415619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868476931, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868476931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868500430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5256763100624084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868500431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868500431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784868508160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190442323684692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868523960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345566391944885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868539745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501973450183868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868555552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469095528125763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868555552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868555553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868579605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5281840562820435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868579606, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868579606, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784868595512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13608399033546448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868612242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401179015636444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868628422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314710557460785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868636368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868636369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868660404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5308187007904053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868660405, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868660405, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784868668538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14038047194480896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868684795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532966375350952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868701246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441476225852966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868717570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527311384677887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868717571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868717571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868741286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5339008569717407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868741286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868741286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784868757209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352696567773819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868773441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327954709529877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868789595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237273693084717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868798153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868798153, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868822989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5375076532363892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868822989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868822990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784868830999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256919384002686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868846799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100332021713257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868862360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588321208953857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868878446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233965635299683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868878447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868878447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868902153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.541955292224884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868902153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868902154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784868918192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347806751728058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868934623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029298186302185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868950630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319665014743805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784868958561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868958562, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868982110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5474483966827393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868982111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868982111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784868989918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240768015384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869006063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13086046278476715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869022311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315799832344055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869038899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277721405029297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869038900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869038900, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869062815, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5541642308235168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869062815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869062816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784869078872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220542669296265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869094861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284438967704773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869110855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433212041854858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869119067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869119067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869142250, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5625165700912476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869142250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869142250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784869150492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489949703216553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869166664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12683990597724915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869182742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723568081855774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869198871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317354023456573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869198871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869198872, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869222185, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5717131495475769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869222185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869222185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784869237983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296549290418625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869253908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893636524677277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869270206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840279936790466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869278122, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869278122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869301429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5807201862335205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869301429, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869301429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784869309542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304447501897812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869326143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280917376279831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869342297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130548894405365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869358331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12892493605613708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869358332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869358332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869381288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5919283628463745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869381289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869381289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784869397210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758556008338928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869413646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303061544895172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869429726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337101012468338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869437715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869437716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869461245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6020538806915283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869461245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869461246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784869469755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321471631526947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869486158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734384834766388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869502396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204357028007507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869518327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758326530456543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869518327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869518327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869541412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6135897040367126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869541413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869541413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784869557678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994195520877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869573743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094595074653625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869590510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889507412910461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869598917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869598917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869622360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6252811551094055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869622361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869622361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784869630541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281415820121765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869646785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282893568277359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869663396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890596687793732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869679872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12817367911338806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869679873, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869679873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869703484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6339766979217529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869703485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869703485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784869720179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12283225357532501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869736204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186761200428009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869751840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12014015018939972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869759744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869759745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869782145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6379403471946716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869782145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869782145, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784869789871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186644971370697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869805872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11871105432510376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869821913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296853214502335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869837705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12075923383235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869837705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869837706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869861310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6574773788452148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869861311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869861311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784869878184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12137813866138458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869894369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130509003996849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869910577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11892740428447723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869918584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869918585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869942392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6787935495376587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869942392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869942393, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784869950300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192016750574112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869966566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925527453422546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869982934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364169418811798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869998828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172865778207779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784869998920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784869998921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784870023005, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7068197131156921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784870023005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784870023005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784870039266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590640246868134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870055252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621499806642532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870071402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903821885585785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870079309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870079310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870101957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7287963032722473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870101958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870101958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784870110024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302515864372253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870126603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138143241405487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870142982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225545406341553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870159570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345705389976501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870159570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870159571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870182881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7536516189575195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870182881, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870182881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784870199187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255127936601639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870215388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129208654165268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870231758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11543719470500946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870239889, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870239890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870262916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7615731954574585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870262916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870262917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784870270876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521388053894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870287529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244302988052368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870303818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062399223446846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870320032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990236699581146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870320032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870320033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870344418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766869843006134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870344419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870344419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784870360928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345174908638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870377098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027105897665024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870393201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867815464735031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870401093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870401093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870424286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685171961784363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870424287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870424287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784870432445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924504697322845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870448728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034150421619415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870465199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133092641830444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870481145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492929816246033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870481146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870481146, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870505723, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699735164642334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870505723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870505723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784870523317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435538530349731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870540144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844556033611298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870556877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553098261356354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870565198, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870565199, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870587663, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689064145088196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870587664, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870587664, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784870595867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447051167488098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870612877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394639313220978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870629506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145921051502228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870645934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279840767383575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870645934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870645935, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870670178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698440551757812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870670178, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870670178, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784870687049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717913299798965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870703861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11732777953147888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870720690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010465025901794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870729056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870729057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870751586, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703806757926941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870751586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870751587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784870760003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478739976882935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870776738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481395363807678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870793355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053420603275299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870810471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400791466236115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870810472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870810473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870833463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715024352073669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870833464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870833464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784870849457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648521780967712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870865901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529173165559769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870882056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546055436134338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870890183, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870890184, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870913238, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719013094902039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870913238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870913238, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784870921380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347109287977219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870938023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097723990678787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870954472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698778927326202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870971148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121751219034195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784870971149, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784870971149, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784870993876, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722285985946655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784870993876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784870993876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784871010285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628720372915268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871026390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061728447675705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871042670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152549296617508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871050641, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871050641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871073461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725415825843811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871073462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871073462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784871081701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204286128282547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871098239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10106198489665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871115001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11092683672904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871131032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366864502429962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871131032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871131033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871155178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730610370635986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871155179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871155179, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784871171696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571674257516861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871188289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607373714447021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871204559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652877390384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871212977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871212977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871236192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733955383300781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871236193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871236193, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784871244495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678546130657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871260819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931115806102753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871277546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751073062419891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871294264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042754054069519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871294264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871294265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871317704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734543085098267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871317705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871317705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784871334029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768401622772217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871350952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792233794927597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871367324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385309159755707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871375451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871375452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871398771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731227278709412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871398772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871398772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784871406837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882290452718735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871423320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702510178089142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871439693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151189357042313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871456268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794492065906525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871456269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871456269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871479336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734448909759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871479337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871479337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784871495863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052134782075882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871511945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156769305467606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871529036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798138380050659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871537529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871537529, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871561306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739500999450684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871561307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871561307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784871569493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398925840854645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871585726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904269874095917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871602266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385751515626907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871618636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014744639396667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871618637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871618637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871642780, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738193273544312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871642780, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871642781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784871659217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616227239370346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871676087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082400530576706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871692699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777496546506882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871700837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871700837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871724794, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742146849632263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871724795, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871724795, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784871733682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965999960899353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871750608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435089260339737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871767439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172356456518173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871784219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256837099790573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871784220, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871784220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871807956, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734901309013367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871807957, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871807957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784871824533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966978967189789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871841560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340569704771042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871858417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017830669879913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871866810, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871866811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871891746, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738568186759949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871891746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871891747, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784871900056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044783145189285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871917396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885351896286011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871934089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259768158197403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871950927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080705374479294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784871950928, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871950928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871974307, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742443084716797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871974307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871974307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784871991152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11015733331441879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872008237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224915087223053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872025224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663895100355148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872033714, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872033715, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872057454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745472192764282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872057454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872057455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784872065626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143219918012619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872082831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732045024633408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872099350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087919980287552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872115787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663136839866638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872115788, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872115788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872139007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751693725585938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872139007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872139008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784872155433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104435034096241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872171802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063511073589325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872188509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491480678319931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872196934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872196935, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872220288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753638029098511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872220288, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872220289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784872228635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828186571598053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872244915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302504152059555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872261981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468199104070663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872278634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687203705310822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784872278635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872278635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872301789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775496780872345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872301790, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872301790, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784872317997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002575814723969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed898059771.log b/recommendation_v4/rcp_logs/gbs_32768/seed898059771.log new file mode 100644 index 000000000..de0ddba2a --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed898059771.log @@ -0,0 +1,701 @@ +:::MLLOG {"namespace": "", "time_ms": 1784923825817, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784923825817, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784923849075, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "seed", "value": 898059771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784923849076, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784923849124, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784923849125, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784924770411, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784924770412, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784924770759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784924909414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851618766784668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924924846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849464058876038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924939347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846668601036072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784924946461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924946462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924969730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500247597694397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924969730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924969730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784924976992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384524405002594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784924991245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845795392990112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925006229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848111033439636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925021048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845863938331604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925021048, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925021049, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925043771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004059672355652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925043772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925043772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784925058584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381346881389618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925073366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822314143180847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925088101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815061748027802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925095263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925095264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925118146, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006042718887329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925118147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925118147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784925125753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818307220935822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925140463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828210532665253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925155392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138348788022995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925170364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796043395996094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925170364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925170365, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925192883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008304119110107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925192884, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925192884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784925207279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810153305530548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925221826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741692900657654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925236451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785940408706665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925243707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925243708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925266860, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501090943813324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925266861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925266861, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784925273885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696201145648956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925288744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716261088848114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925303145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689057528972626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925317885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368727684020996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925317886, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925317886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925341336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013877153396606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925341337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925341337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784925355548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705477118492126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925370167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370309740304947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925385493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629266619682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925392856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925392857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925415822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016984939575195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925415823, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925415823, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784925423214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708919286727905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925438932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361178159713745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925453637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643459975719452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925469116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369544416666031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925469117, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925469117, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925491803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020865797996521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925491804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925491804, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784925506435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611721992492676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925521699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477958738803864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784925536360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654008507728577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925543663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925543663, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925567017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024632215499878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925567017, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925567018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784925574255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329861581325531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925589248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454130291938782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925603787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487952947616577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925618356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505491614341736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925618425, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925618426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925642345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028306245803833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925642346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925642346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784925656834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368241488933563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925671359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447807729244232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925685943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351320892572403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925694095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925694095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925716771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032281279563904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925716772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925716772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784925723908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565343618392944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925738509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489462435245514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925753488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184131681919098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925769004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453030586242676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925769005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925769006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925791416, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037698149681091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925791417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925791417, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784925806333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374684751033783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925821647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352718472480774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925837296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348884552717209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925844884, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925844884, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925868512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046960115432739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925868513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925868513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784925875932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637982308864594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925891009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334618180990219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925906550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425351679325104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925921397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320488601922989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925921462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925921462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925944864, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5056262016296387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925944864, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925944864, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784925959512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287048041820526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925975028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13123157620429993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925989792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307799398899078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784925997251, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784925997252, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926020101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5066063404083252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926020103, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926020103, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784926027461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138504296541214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926042357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108564913272858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926057835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986746430397034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926072465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029572367668152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926072465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926072466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926095511, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077484250068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926095511, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926095511, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784926110357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278742134571075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926125445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983572483062744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926141015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130569726228714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926148446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926148447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926171268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5089661478996277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926171269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926171269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784926178781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315373033285141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926193639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515204191207886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926208963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331835687160492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926224942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561052083969116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926224943, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926224944, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926247975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5103926658630371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926247975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926247976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784926262779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524380326271057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926277677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255837559700012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926292651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322246074676514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926300055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926300056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926323797, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118592381477356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926323797, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926323798, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784926332195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630619645118713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926347479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517722487449646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926362791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362120509147644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926378297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638201355934143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926378298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926378298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926401775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5133873820304871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926401776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926401776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784926417776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495218753814697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926433031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088847696781158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926448046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624048233032227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926455763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926455764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926479263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5148209929466248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926479263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926479263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784926486803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597680628299713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926502361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243933022022247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926517387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271787762641907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926532336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082416355609894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926532337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926532337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926555160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160877108573914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926555161, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926555161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784926570230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105623424053192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926586059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827309012413025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926601131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948867678642273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926608548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926608549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926631493, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.517541229724884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926631493, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926631494, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784926638938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939569354057312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926654115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479766249656677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926669323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460421562194824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926685626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304318904876709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926685626, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926685627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926709251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5193918347358704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926709252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926709252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784926724226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281765252351761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926739284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200750946998596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926754541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355375349521637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926762275, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926762275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926786201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5214159488677979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926786201, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926786202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784926793796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665568828582764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926809740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355785608291626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926824897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205572962760925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926839822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310458779335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926839822, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926839823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926863414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238081812858582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926863415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926863415, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784926878498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955807149410248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926893539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248397409915924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926909207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353769600391388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926916807, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926916808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926939784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5268151760101318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926939785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926939785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784926947301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197296857833862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926962652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342124223709106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926977965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493192195892334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926993375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460201025009155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784926993375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784926993375, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927017071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5301899909973145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927017071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927017072, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784927032353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603439927101135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927048414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396748900413513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927064054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311733305454254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927071651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927071651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927095212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5341899394989014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927095213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927095213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784927103197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14035245776176453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927119019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525976240634918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927134868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436466455459595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927150836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351659744977951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927150836, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927150837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927174644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.538504421710968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927174645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927174645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784927190087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520148396492004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927205622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317759335041046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927221172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235794007778168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927229396, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927229397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927253410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5434097647666931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927253411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927253411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784927261207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324964016675949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927276643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084498047828674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927291868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577058911323547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927307304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322665959596634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927307304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927307305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927331651, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.549701452255249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927331651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927331652, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784927347133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463589549064636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927363266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300952285528183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927378774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13174594938755035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927386479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927386480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927410139, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5583450198173523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927410140, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927410140, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784927417627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205185532569885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927433035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306046098470688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927448642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270357251167297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927464476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242322206497192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927464477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927464477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927488526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.570210874080658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927488527, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927488527, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784927503891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167372345924377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927519202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277436912059784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927534278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336391419172287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927542121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927542121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927565908, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5821228623390198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927565909, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927565909, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784927573689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421624898910522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927589116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12619149684906006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927605066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262417584657669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927620744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205039501190186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927620745, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927620745, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927644303, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5937678217887878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927644304, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927644304, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784927659547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12834595143795013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927674660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274568885564804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927690267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12695084512233734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927697761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927697761, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927721317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6025128364562988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927721317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927721318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784927729098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129136323928833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927745203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584933638572693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927760668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840625643730164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927776072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266225427389145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927776072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927776073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927799374, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6101095676422119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927799375, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927799375, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784927814569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12500397861003876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927830277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12816780805587769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927845720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303817480802536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927853440, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927853440, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927876641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6161215305328369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927876642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927876642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784927884924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290951669216156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927900893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12388870120048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927916777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12900137901306152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927932450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12309715151786804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927932450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927932451, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927955928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6213796734809875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927955929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927955929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784927971865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12595142424106598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784927987579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714475393295288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928004075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.125588059425354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928012080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928012081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928035151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6272590160369873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928035153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928035153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784928043009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12501442432403564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928058691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498724460601807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928074571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12442117929458618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928090518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12441711127758026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928090519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928090519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928113871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6306261420249939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928113872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928113872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784928130421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677435040473938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928146120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765661835670471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928161445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538287252187729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928169255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928169256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928192643, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6335803270339966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928192643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928192644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784928200191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792019754648209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928215659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475792527198792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928231299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11987549066543579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928246814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172080934047699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928246815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928246815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928270388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6394566893577576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928270388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928270388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784928286940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195121556520462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928302602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107531189918518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928318168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165068969130516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928325841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928325841, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928349367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6472029089927673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928349368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928349368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784928357026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178789734840393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928372869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11788955330848694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928388631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242390424013138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928403917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633750051259995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928403987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928403988, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928428317, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6570914387702942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928428318, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928428318, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784928444107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159684956073761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928459719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594419181346893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928475503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105942353606224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928483234, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928483234, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928506695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6646050214767456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928506696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928506696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784928514447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288287490606308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928530242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11517022550106049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928545971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341430991888046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928562005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583206802606583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928562006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928562006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928585470, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6773422956466675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928585471, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928585471, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784928601059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147501990199089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928616536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11441762745380402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928632283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11793440580368042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928639956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928639957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928662829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6881502866744995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928662830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928662830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784928670551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916250199079514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928686601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594663560390472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928702307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993434488773346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928718233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11579741537570953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928718233, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928718234, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928741871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6994078159332275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928741871, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928741872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784928757737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11797631531953812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928773178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677858978509903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928788683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446114629507065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928796175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928796176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928818999, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.705192506313324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928818999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928819000, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784928826709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421656608581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928842331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11741218715906143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928858214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11570577323436737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928873472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213439702987671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928873473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928873473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928897624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7141785621643066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928897625, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928897625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784928914641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11967889219522476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928930721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12333181500434875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928946908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169323116540909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784928954936, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928954937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928977310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7306665182113647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928977311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928977311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784928985174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12018358707427979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929001404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12000530958175659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929017383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11820365488529205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929033165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773968487977982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929033166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929033166, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929056965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7390012145042419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929056966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929056966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784929072857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12320350855588913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929088455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032131850719452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929104270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464203149080276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929112161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929112162, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929134961, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.745743989944458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929134962, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929134962, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784929142852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12216303497552872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929158805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11865490674972534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929174666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490596830844879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929191101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936305671930313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929191102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929191102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929213789, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7517828941345215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929213789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929213790, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784929229109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118147075176239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929244771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101673543453217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929260242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061378568410873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929268020, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929268021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929290907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7550656795501709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929290907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929290908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784929298779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096469834446907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929314887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520106345415115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929330782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192522943019867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929347046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776698380708694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929347047, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929347047, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929369744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7601944804191589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929369745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929369745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784929385474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154051870107651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929400822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128106713294983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929416427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718280076980591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929424017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929424018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929447704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7604387998580933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929447704, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929447705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784929455506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707204043865204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929471336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470281541347504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929487452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618468165397644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929503023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947529971599579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929503023, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929503024, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929526573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7623239755630493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929526574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929526574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784929542683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038082838058472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929558546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033276468515396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929573934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976121574640274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929581920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929581921, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929604719, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7642564177513123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929604720, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929604720, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784929612656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947246104478836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929628136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022871807217598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929644003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110089972615242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929659940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725268721580505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929659940, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929659940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929682734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7650846838951111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929682735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929682735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784929698252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897781699895859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929714649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084393411874771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929730410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860949009656906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929738100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929738100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929760821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661783695220947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929760822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929760822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784929768469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129896491765976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929784090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901700705289841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929799376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256872862577438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929815017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932882130146027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929815018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929815018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929837702, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661504745483398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929837703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929837703, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784929853509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675202310085297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929868944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424366593360901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929885349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095624715089798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929893437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929893437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929915874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7668023705482483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929915875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929915875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784929923816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602317750453949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929939393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007661446928978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929955230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616943031549454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929970964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212486028671265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784929970965, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784929970965, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784929994388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7674172520637512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784929994388, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784929994389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784930010353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732746124267578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930026630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996231436729431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930042582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958119481801987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930050427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930050428, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930074203, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677519917488098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930074204, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930074204, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784930082921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101046949625015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930099202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526267975568771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930115365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349549144506454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930131476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626327782869339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930131477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930131477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930154052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685096263885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930154052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930154053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784930169936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139160394668579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930186367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153145283460617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930202707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158185452222824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930210826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930210826, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930234328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769062340259552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930234329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930234329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784930242305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214864253997803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930259221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005795001983643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930275339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377231776714325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930291532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216118931770325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930291532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930291533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930314183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694895267486572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930314184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930314184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784930330391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090710759162903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930346960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131872832775116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930363417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11767882108688354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930371625, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930371626, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930395310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.769891083240509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930395311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930395311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784930403171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552394926548004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930419837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812386125326157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930435778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247902572154999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930451641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793086141347885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930451642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930451642, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930474738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701855301856995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930474738, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930474739, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784930490509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592330992221832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930506328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686737298965454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930522502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664002597332001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930530633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930530634, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930554811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703453898429871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930554812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930554812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784930562762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895249247550964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930578372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373222082853317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930595251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513504594564438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930611412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790741443634033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930611413, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930611414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930634533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709788084030151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930634534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930634534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784930650177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105334222316742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930666221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475845634937286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930682162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409890860319138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930690095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930690096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930713930, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714491486549377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930713931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930713931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784930721974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509669035673141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930738475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033371239900589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930754531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474989354610443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930770983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400542616844177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930770983, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930770984, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930794610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715696692466736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930794611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930794611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784930810603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781586915254593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930826541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874763876199722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930842660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239779204130173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930850706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930850707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930873909, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718785405158997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930873910, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930873910, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784930881905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019010990858078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930897953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103213042020798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930914213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670678317546844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930930658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034335270524025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930930658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930930659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930953811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722553014755249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930953812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930953812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784930969590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084270179271698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784930986500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429082065820694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931002442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698859393596649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931010222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931010223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931034038, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724200487136841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931034039, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931034039, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784931041945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422153770923615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931057834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090250164270401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931073684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343965888023376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931090113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761374235153198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931090114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931090114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931113659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727489471435547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931113660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931113660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784931130013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026747077703476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784931145866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736606270074844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.398333333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed942014509.log b/recommendation_v4/rcp_logs/gbs_32768/seed942014509.log new file mode 100644 index 000000000..fe6b60b53 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed942014509.log @@ -0,0 +1,675 @@ +:::MLLOG {"namespace": "", "time_ms": 1784950456391, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784950456391, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784950479613, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784950479615, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "seed", "value": 942014509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784950479616, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784950479631, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784950479632, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784951311027, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784951311027, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784951311354, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784951630015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938525319099426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951646073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392778754234314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951661234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391793042421341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784951668638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951668639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951713176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4997166693210602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951713177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951713177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784951720632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905693590641022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951735373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391364336013794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951750718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390824317932129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951765761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889843225479126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951765761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951765762, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951788363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49986180663108826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951788364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951788364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784951803493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390383243560791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951818833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388559341430664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951834036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886678218841553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951841392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951841393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951864659, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000264048576355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951864660, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951864660, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784951872479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878445327281952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951887776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870707154273987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951903267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863611221313477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951918497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851848244667053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951918497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951918498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951941077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002193450927734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951941078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951941078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784951956045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850121200084686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951971035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821986317634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951986216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833360373973846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784951993801, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784951993802, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952016557, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500444769859314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952016558, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952016558, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784952023916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788744807243347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952039433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774146139621735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952054279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760662078857422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952069445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375463306903839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952069446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952069446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952092418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006869435310364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952092419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952092419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784952107219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757741451263428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952122487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374356895685196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952138419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696229457855225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952145937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952145937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952169152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009583234786987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952169153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952169153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784952176730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374688744544983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952192933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668303191661835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952208359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687802851200104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952224644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727916777133942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952224644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952224645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952247092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013055205345154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952247093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952247093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784952262278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365201473236084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952278184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135393887758255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784952293332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368338018655777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952300920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952300920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952323258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016603469848633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952323259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952323259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784952330860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382910192012787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952346229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510912656784058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952361052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525255024433136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952376074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543306291103363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952376144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952376144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952399907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020328164100647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952399907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952399908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784952415006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424141705036163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952430058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484102487564087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952445099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535813987255096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952453469, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952453470, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952475854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024929642677307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952475855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952475855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784952483316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13578665256500244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952498412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507810235023499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952513871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132317453622818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952529784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347334384918213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952529784, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952529785, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952552036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030809640884399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952552037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952552038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784952567335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391870260238647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952582892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525399565696716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952599251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505718111991882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952607119, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952607119, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952630022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039742588996887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952630022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952630023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784952637844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363483965396881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952653670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358965516090393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952669650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420884311199188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952685151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205666840076447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952685215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952685216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952708398, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049456357955933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952708399, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952708399, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784952723682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328895539045334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952739941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133732974529266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952755373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318605720996857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952763187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952763187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952786528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5060123205184937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952786529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952786529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784952794188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836802542209625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952809827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105586171150208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952825859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988857924938202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952841235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13041047751903534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952841235, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952841236, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952863870, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072387456893921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952863871, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952863871, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784952879262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275563716888428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952894889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979170680046082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952910980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059017062187195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952918634, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952918635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952941714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086020231246948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952941715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952941715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784952949406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157521188259125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952964788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514301180839539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952980702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311049342155457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952996877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356322169303894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952996877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784952996878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784953019816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.510015070438385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784953019816, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784953019816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784953035316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523483276367188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953050857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265900313854218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953066535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322661817073822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953074316, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953074316, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953097536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5113710165023804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953097537, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953097537, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784953106372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635943830013275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953122415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135268896818161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953138342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621237874031067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953154661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634896278381348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953154661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953154661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953178065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.512747049331665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953178066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953178066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784953194948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348572075366974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953211148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307659149169922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953226920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362570822238922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953235011, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953235012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953258596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5141309499740601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953258597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953258597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784953266476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595065474510193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953283022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252303004264832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953298664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13269659876823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953313893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082928955554962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953313893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953313894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953336894, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5155054926872253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953336894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953336895, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784953352506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100923597812653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953368756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382891833782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953384391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956582009792328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953392205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953392206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953414884, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5170167684555054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953414885, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953414885, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784953422569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293007880449295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953438291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347917914390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953454128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345607042312622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953470649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043645024299622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953470650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953470650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953493244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5188102126121521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953493245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953493245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784953508636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12815743684768677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953524090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201385736465454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953539771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356455236673355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953547756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953547756, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953570396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5206784009933472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953570397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953570397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784953578143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665953278541565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953594278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361874222755432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953609907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321638971567154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953625457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310407519340515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953625457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953625457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953648307, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5227980017662048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953648308, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953648308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784953663997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951189279556274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953679710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132567897439003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953696151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545721769332886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953704054, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953704055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953727219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5253384709358215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953727220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953727220, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784953734895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319354623556137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953750611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333875209093094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953766157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501572608947754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953781882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464263081550598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953781882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953781883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953805800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5281730890274048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953805801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953805801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784953821847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360304355621338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953838396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395845890045166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953854433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330866515636444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953862292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953862292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953885880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5313276648521423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953885880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953885880, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784953893976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403176188468933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953910119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352604180574417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953926426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442742824554443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953943055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352378875017166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953943055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953943056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953966157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5347365736961365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953966158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953966158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784953982380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526329398155212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784953998613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329443335533142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954014828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323557198047638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954023472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954023472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954047257, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5387015342712402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954047258, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954047258, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784954055228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325884312391281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954070931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082589209079742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954086493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13591116666793823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954102399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323847472667694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954102400, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954102400, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954125331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.543684720993042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954125332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954125332, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784954141214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471239805221558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954157906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302986741065979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954173907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319255828857422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954181878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954181879, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954204931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5503340363502502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954204932, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954204932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784954212700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323852837085724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954228661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307922601699829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954244731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314031064510345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954261331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281212747097015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954261332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954261332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954284865, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.559302806854248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954284866, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954284866, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784954300797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204576075077057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954316599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282271146774292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954332241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13405659794807434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954340324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954340324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954363609, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5698170065879822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954363610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954363610, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784954371731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464201986789703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954387729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12685194611549377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954404214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12686079740524292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954420502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13286302983760834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954420503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954420503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954443536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5813202857971191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954443537, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954443537, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784954459398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292615532875061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954475522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856006622314453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954491867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12787222862243652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954499808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954499808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954523459, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5926159024238586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954523460, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954523461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784954531615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008570671081543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954548152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12742114067077637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954563989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300102174282074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954579720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796533107757568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954579721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954579721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954602977, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6032059788703918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954602978, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954602978, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784954618680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651316821575165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954634829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944066524505615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954650742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251934945583344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954658690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954658691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954682769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6120533347129822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954682770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954682770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784954691160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309661865234375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954707920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600508332252502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954724306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308455467224121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954740536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12590740621089935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954740537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954740537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954764399, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6195132732391357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954764399, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954764400, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784954781025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12831273674964905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954797413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295386403799057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954814479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751233577728271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954822702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954822703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954846572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6266418099403381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954846572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954846572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784954854688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12671127915382385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954870685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670651078224182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954886922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12698805332183838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954903335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12633486092090607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954903336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954903336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954927105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.632881760597229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954927106, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954927106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784954943636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11993734538555145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954960017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969032883644104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954976160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773760616779327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784954984250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784954984251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784955007423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6400097608566284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784955007424, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784955007424, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784955015302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11992330849170685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955031561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11675269901752472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955047786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12138229608535767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955063771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832931637763977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955063771, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955063771, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955088192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6476168036460876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955088193, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955088193, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784955105000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199447512626648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955121035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111164316534996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955137004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694562435150146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955144838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955144838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955168145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6602447032928467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955168146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955168146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784955176007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11762797832489014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955192305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814603954553604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955208525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130245178937912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955224357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11668012291193008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955224431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955224431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955248898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6784640550613403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955248898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955248898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784955265450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540035158395767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955281633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495864391326904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955298080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799530148506165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955306170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955306171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955330169, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7081494927406311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955330169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955330169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784955338321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112007275223732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955354782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348062008619308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955371068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113385260105133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955387572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355845630168915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955387573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955387573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955411843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7476333975791931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955411843, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955411844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784955427886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147540807723999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955443888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088179796934128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955460175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454156786203384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955468196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955468196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955491457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7621630430221558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955491458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955491458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784955499564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373278707265854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955516290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199760437011719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955532584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550068318843842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955549171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976584255695343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955549172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955549172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955573253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619096040725708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955573254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955573254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784955589444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299178749322891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955605439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008986085653305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955621380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864154994487762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955629171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955629171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955652326, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7626489400863647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955652327, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955652327, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784955660455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915185511112213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955676764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044737696647644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955693265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118411049246788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955709273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464973002672195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955709274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955709274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955733520, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628002166748047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955733521, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955733521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784955751033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454451829195023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955767552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11876041442155838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955784200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563084274530411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955792390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955792390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955816323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7621578574180603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955816323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955816324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784955824471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498970538377762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955841310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427876353263855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955857803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222153902053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955874026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336449533700943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955874027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955874028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955898166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634860277175903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955898166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955898166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784955914794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792377382516861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955930983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11716579645872116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955947517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058290302753448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784955955700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955955700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955979158, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.76301509141922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955979159, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955979159, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784955987455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151614785194397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956004022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526508629322052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956020413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156243830919266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956037390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439623147249222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956037390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956037391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956061045, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7655073404312134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956061046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956061046, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784956077172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652606934309006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956093670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597768425941467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956109993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580595582723618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956118206, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956118207, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956141914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7660907506942749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956141914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956141915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784956150078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435639321804047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956166888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129070073366165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956183441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774901509284973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956200295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314158886671066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956200295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956200296, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956223138, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670086026191711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956223139, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956223139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784956239623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069294661283493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956255508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634833574295044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956271614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200981050729752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956279471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956279472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956302217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670903205871582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956302218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956302218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784956310361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026434674859047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956326735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171990841627121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956343369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174603551626205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956359580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041506752371788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956359581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956359581, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956383365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682419419288635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956383365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956383365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784956400014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640476644039154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956416811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669327527284622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956433041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722540318965912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956441454, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956441455, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956464192, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683331966400146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956464192, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956464192, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784956472479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726846754550934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956488761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986816346645355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956505338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772515088319778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956521840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430725663900375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956521840, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956521841, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956545057, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7684311270713806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956545057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956545057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784956560979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744550824165344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956577781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793745517730713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956594158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435591638088226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956602269, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956602269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956625374, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673854827880859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956625374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956625375, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784956633380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898106545209885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956649821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737067461013794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956666016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189060658216476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956682606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854028910398483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956682607, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956682607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956705883, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689808011054993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956705883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956705884, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784956722492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545680671930313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956738706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101863332092762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956755552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086369976401329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956764004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956764005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956786678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7699291706085205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956786679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956786679, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784956794694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465662181377411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956810695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986721724271774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956827148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141803190112114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956843457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111834645271301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956843457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956843458, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956867292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698155641555786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956867293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956867293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784956883688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646301507949829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956900430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891763865947723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956916954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868026316165924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956924944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956924945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956948582, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7693923115730286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956948583, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956948583, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784956957276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033798009157181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956974214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486578732728958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784956991163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282214522361755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957007969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303339898586273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957007970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957007970, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957031071, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692562937736511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957031071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957031071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784957047694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031508445739746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957064659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396831274032593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957081639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091317981481552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957090037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957090037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957113316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697288393974304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957113317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957113317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784957121591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114037036895752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957139141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027653515338898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957155920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304263025522232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957172598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125653982162476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957172598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957172598, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957195029, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704236507415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957195029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957195030, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784957211809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016608029603958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957228793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305087059736252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957245778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720065027475357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957254223, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957254223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957277321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770533561706543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957277321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957277322, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784957285475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11493484675884247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957302742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080770194530487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.331666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957319230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020095944404602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957335612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734478384256363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957335612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957335613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957358972, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709159255027771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957358973, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957358973, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784957375318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520628839731216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957391484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688610374927521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957408094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560768842697144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957416453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957416453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957438785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710500955581665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957438786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957438786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784957447079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872358083724976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957463281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362883657217026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957480572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498858988285065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957497020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793419182300568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.631666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957497021, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957497021, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957519267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714108824729919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957519268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957519268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784957535257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092955082654953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957551615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039704754948616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957567789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353685915470123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957575920, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957575920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957598781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715893983840942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957598782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957598782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784957607108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451418906450272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957623953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032935231924057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957640179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144774928689003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957656907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395635664463043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957656907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957656908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957679842, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715200185775757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957679842, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957679843, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784957696443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757070779800415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957712793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759414732456207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957729012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020493358373642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957737143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957737144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957760134, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771209716796875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957760135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957760135, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784957768347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225065052509308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784957784731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082433372735977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.031666666666666e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_32768/seed996913233.log b/recommendation_v4/rcp_logs/gbs_32768/seed996913233.log new file mode 100644 index 000000000..2a6e9148e --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_32768/seed996913233.log @@ -0,0 +1,617 @@ +:::MLLOG {"namespace": "", "time_ms": 1784916338426, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784916338427, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784916362778, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 32768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "seed", "value": 996913233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784916362781, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 4e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784916362803, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784916362804, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784917180038, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784917180039, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784917180377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784917537545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884864747524261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.1666666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917553356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883957266807556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.4999999999999995e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917568349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878655433654785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.833333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784917575814, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917575814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917619029, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015365481376648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917619030, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917619030, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784917626519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388218253850937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3166666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917641331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867776095867157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917656732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858988881111145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9833333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917671911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385660022497177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917671911, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917671911, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917696269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016977190971375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917696270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917696270, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784917711464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853825628757477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917726736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138581782579422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917742025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844941556453705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917749458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917749458, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917774195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018845200538635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917774196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917774196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784917782171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844358921051025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917797553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840222358703613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917813269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832490146160126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917828779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818082213401794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917828780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917828780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917854538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020955801010132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917854539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917854539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784917869617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825136423110962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917884674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789550960063934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.3166666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917901553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799332082271576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6499999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917909184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917909185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917934840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023759007453918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917934841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917934841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784917942279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736926019191742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917958149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754421472549438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917973296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725632429122925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917988850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726945221424103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.983333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784917988851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784917988851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918016296, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026543140411377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918016296, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918016297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784918031323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735778629779816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.316666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918050170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722500205039978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918066620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366882622241974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918074511, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918074512, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918100140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5029589533805847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918100141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918100141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784918107970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736268877983093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918124471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365812122821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.649999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918140281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675464689731598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.983333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918156848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716955482959747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.316666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918156848, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918156849, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918183694, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033390522003174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918183694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918183694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784918199483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646015524864197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.65e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918215714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353917121887207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.983333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784918231577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674575090408325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918239279, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918239280, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918263031, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037326216697693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918263032, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918263032, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784918270835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384465873241425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918287441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506847620010376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918303011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531552255153656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918318559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541756570339203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918318652, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918318652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918346673, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5041405558586121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918346674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918346674, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784918362252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422803580760956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918379771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494768738746643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918395538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354362815618515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918404432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918404432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918431742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045881867408752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918431743, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918431743, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784918439542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588431477546692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918455292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511045277118683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918471155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251593708992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918488022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471272587776184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.3983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918488022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918488022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918514313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5051766633987427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918514314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918514314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784918530613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402649760246277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918546853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525840640068054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918563795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135014146566391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918572110, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918572110, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918598343, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061303973197937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918598344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918598344, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784918606268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634821772575378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918622292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351459801197052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918639242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343356966972351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918655321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228002190589905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918655388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918655388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918682392, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071397423744202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918682392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918682393, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784918698323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299311697483063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918715058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313813328742981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.6983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918731613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312095403671265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918739767, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918739767, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918766014, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5082727074623108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918766015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918766015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784918774029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831080496311188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.7649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918790157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131221741437912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918807070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989598512649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918823340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042806088924408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.8649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918823341, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918823341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918850110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095977783203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918850111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918850111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784918866002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327870786190033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918881903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976966798305511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918898840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305113136768341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918907015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918907016, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918933428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5110715627670288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918933429, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918933429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784918941437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315292865037918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.9983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918957468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503853976726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918973651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331634521484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918991009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562193512916565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784918991010, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784918991010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784919017539, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5126305818557739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784919017539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784919017540, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784919033929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513460755348206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919050238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236509263515472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.1649999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919066328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324055075645447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1983333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919074338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919074338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919098626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.514167308807373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919098626, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919098626, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784919108105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363925188779831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919125123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524962961673737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919141646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362456977367401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919158136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642486929893494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919158137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919158137, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919183971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5156493782997131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919183972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919183972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784919202153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478854298591614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.3649999999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919219312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078776001930237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919235809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611596822738647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919244249, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919244249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919270000, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5171640515327454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919270000, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919270000, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784919278076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594216108322144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919295137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243165612220764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919311740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255521655082703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919328303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081327080726624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919328304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919328304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919354250, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5187360644340515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919354250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919354250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784919370335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310792863368988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919387917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812652230262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919404753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294945627450943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919412911, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919412912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919438913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.520534873008728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919438914, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919438914, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784919446980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12933844327926636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919463676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347765326499939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919479937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462845981121063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919497170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039666414260864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919497171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919497171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919523958, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5225886702537537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919523959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919523959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784919540560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812545895576477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919556962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13198766112327576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919573583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543885946273804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919581849, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919581850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919607251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5248034000396729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919607252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919607252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784919615458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658864796161652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919633063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335182636976242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919649724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201770186424255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919666249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309453248977661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919666249, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919666250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919693087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5273799896240234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919693088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919693088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784919709703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294645518064499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919725895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247615098953247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919743387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541388511657715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919751721, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919751722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919777528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5302093625068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919777529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919777529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784919785809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185319304466248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919802664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340872526168823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919819338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496878743171692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919835743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456059992313385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919835743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919835743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919862150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.533427894115448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919862150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919862151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784919878422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358908861875534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919895887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390816748142242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919912942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305188715457916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.3650000000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919921390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919921390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919948036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5368315577507019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919948037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919948037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784919956766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14043119549751282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919973693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534612953662872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784919991423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442517817020416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.4649999999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920008745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352335810661316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920008746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920008747, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920034620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5406181812286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920034620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920034620, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784920051586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515880703926086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920068459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323073089122772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.5649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920085129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225868344306946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.5983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920094170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920094171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920122352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5448633432388306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920122353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920122353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784920130972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245484232902527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.6316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920147873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078488409519196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920164761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13578557968139648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.6983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920181552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219434022903442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.7316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920181553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920181553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920207698, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5499485731124878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920207698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920207699, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784920224426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458700478076935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920242283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301666647195816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.7983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920259270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179247081279755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920267650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920267651, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920293674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5563074946403503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920293675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920293675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784920302004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219869136810303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.8649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920318606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060089945793152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920335259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329658478498459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920352439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245044648647308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920352440, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920352441, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920379283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5642351508140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920379283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920379284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784920396235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188835978507996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920413082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799251079559326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920429970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402059674263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.0649999999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920438486, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920438487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920465170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5733521580696106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920465171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920465171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784920473761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13461050391197205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.0983333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920490384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12655183672904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.1316666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920507573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12683254480361938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.1649999999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920524731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281089067459106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.1983333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920524732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920524732, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920551481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5839505195617676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920551482, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920551482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784920568246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12915225327014923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920585144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12850001454353333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920602463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780198454856873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.2983333333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920610618, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920610618, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920636718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5921359658241272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920636719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920636719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784920645169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299773007631302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3316666666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920663013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273553967475891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.3649999999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920680261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982575595378876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920697305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809477746486664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.4316666666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920697306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920697306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920722951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6033560633659363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920722951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920722951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784920739795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12674446403980255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920757194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951405346393585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4983333333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920773816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254496455192566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.5316666666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920782117, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920782118, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920808604, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6141943335533142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920808604, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920808604, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784920817739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107900321483612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920835479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126066654920578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920852951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308017373085022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.6316666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920870327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613433599472046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920870328, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920870328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920896718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.625633955001831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920896719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920896719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784920914155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12858355045318604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920931061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972989678382874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920949105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750297784805298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.7649999999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784920958106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920958107, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920984536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.636995792388916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920984536, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920984536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784920993291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12697193026542664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921010492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659117579460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921027923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12729662656784058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921044973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12621372938156128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.898333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921044974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921044974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921071876, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6481412649154663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921071876, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921071876, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784921089536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12040795385837555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921106941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975416541099548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921123793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814992129802704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.998333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921132281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921132281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921158898, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6511724591255188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921158899, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921158899, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784921167313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975765228271484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921184261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673430353403091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921201299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214437410235405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.098333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921217819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11865557730197906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921217819, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921217819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921244528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6733489036560059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921244529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921244529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784921262512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12017570436000824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921279390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151523888111115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.198333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921296469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698942631483078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921304896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921304897, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921331323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6972360610961914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921331323, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921331323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784921339663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787327378988266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921356861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780139058828354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.298333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921373645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274323612451553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921389999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11612795293331146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921390067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921390067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921418591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7243638634681702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921418591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921418591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784921435868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472000926733017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.398333333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921452936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458882689476013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921470340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080269068479538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921478800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921478801, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921505592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7400403022766113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921505593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921505593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784921514169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267617344856262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.498333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921531552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278803646564484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.531666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921548601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064855754375458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921566249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254417151212692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.598333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921566250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921566250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921594170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7591232061386108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921594170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921594170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784921611286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153199523687363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921628258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041388660669327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921645420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519081890583038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921653829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921653830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921680547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7623952031135559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921680547, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921680548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784921688991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415256768465042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921706444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270906031131744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921723163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637424886226654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921740541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960125923156738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921740541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921740542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921768063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7655860185623169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921768064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921768064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784921785412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356334388256073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921802584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064794659614563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921819820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893280059099197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.931666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921828219, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921828220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921854714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7667389512062073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921854714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921854714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784921863459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951267182826996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921880976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073066294193268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921898418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148597300052643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.031666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921914921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527034103870392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921914922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921914922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921941272, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682448029518127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921941272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921941272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784921959906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472244560718536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921977475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11862578988075256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784921995167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560657620429993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922003965, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922003965, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922030282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696260213851929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922030283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922030283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784922038928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496349424123764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922056682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474613845348358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922074029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205749213695526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.265e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922091053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331313103437424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922091055, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922091055, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922120190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701637148857117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922120190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922120191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784922137930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11766868829727173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922155586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11797455698251724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922173369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042679101228714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922182230, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922182231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922209235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708371877670288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922209235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922209236, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784922218107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558092385530472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922235610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510691791772842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922252728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135758459568024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922271130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404989123344421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922271131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922271131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922297984, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7710654139518738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922297985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922297986, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784922315157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067165657877922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.565e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922332568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058078408241272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922349879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575175285339355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922358582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922358583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922384515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712990045547485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922384515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922384515, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784922393178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038592979311943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922410841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139846593141556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.698333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922427802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748498141765594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922445701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253955960273743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.765e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922445702, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922445703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922472445, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717676758766174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922472445, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922472445, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784922489885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679377615451813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.798333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922506919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647456347942352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922524074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162212699651718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.865e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922532503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922532503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922559191, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721009850502014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922559192, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922559192, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784922567923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283152014017105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.898333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922585405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180187225341797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922603220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140365153551102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.965e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922619864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402914136648178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.998333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922619865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922619865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922646575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721282839775085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922646576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922646576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784922664736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614567995071411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922682460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640344768762589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.065e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922699701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669532418251038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922708602, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922708603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922735094, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724341154098511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922735094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922735094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784922743795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723066329956055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.131666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922761127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976894408464432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.165e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922778777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076837033033371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922796286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464829951524734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.231666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922796287, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922796287, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922822944, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772753119468689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922822945, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922822945, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784922839652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772513598203659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.265000000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922858035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805671662092209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.298333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922875708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436046123504639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.331666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922884403, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922884404, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922910988, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729889154434204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922910988, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922910988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784922919603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914454609155655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.365e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922937080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772612690925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.398333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922954468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176042258739471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.431666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922972148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081547811627388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.465e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784922972148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922972148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922998793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732656598091125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922998794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784922998794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784923016340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049158126115799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.498333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923033172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019463837146759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.531666666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923051573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108269102871418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.564999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923060523, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923060524, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923086693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734629511833191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923086694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923086694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784923095339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447770357131958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.598333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923112515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965184330940247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.631666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923129914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11441800743341446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.664999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923147345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088375747203827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.698333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923147345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923147346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923173947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736234068870544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923173948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923173948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784923191412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647378116846085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.731666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923209210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873275995254517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.764999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923226356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804112255573273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.798333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923234734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923234735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923262475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739405632019043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923262475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923262475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784923272208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994982719421387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.831666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923290333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145445704460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.864999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923308344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243587732315063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.898333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923326210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313311755657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.931666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923326211, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923326212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923353312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737875580787659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923353312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923353313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784923371189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006898432970047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.964999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923389266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387496441602707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.998333333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923407042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076583713293076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.031666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923415719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923415720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923442730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741102576255798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923442730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923442731, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784923451347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090462654829025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.064999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923470355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953503847122192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.098333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923488079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290520429611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.131666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923505877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129438877105713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.164999999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923505878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923505878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923532350, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744260430335999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923532351, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923532351, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784923550168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031082272529602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.198333333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923568182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128053218126297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.231666666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784923585773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11729985475540161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.264999999999999e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed1023733178.log b/recommendation_v4/rcp_logs/gbs_8192/seed1023733178.log new file mode 100644 index 000000000..7052fed72 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed1023733178.log @@ -0,0 +1,1779 @@ +:::MLLOG {"namespace": "", "time_ms": 1785027753977, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1785027753977, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1785027775226, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1023733178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1785027775228, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1785027775229, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1785028699322, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1785028699323, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1785028699641, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1785028821175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844019174575806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1785028833847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13924850523471832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028846936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864125311374664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028860229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13920073211193085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028873660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391812264919281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028887007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896214962005615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028900396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392296403646469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028913955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854539394378662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028927437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886260986328125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028940753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391156166791916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028954514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876685500144958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028968074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872379064559937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028981809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899289071559906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785028995227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867613673210144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785028995228, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785028995228, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029081412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003319978713989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029081413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029081414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029094766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919948041439056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029108425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895992934703827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029122414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386914849281311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029135997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13917577266693115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029149782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906386494636536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029163995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138967365026474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029177624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893288373947144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029191471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885538280010223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029205949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895708322525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029219957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388533115386963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029234448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871565461158752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029248821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866043090820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029262881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388092339038849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029277416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876329362392426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029277417, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029277417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029364601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004019141197205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029364602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029364603, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029378844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859286904335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029393177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870058953762054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029407514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864335417747498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029421989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807161152362823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029436194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833846151828766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029450397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860875368118286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029464595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382833570241928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029479330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812696933746338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029493743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762766122817993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029507992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796959817409515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029522371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381940394639969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029536503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811561465263367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029551703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831716775894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029566385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811111450195312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029566386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029566386, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029653076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004539489746094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029653077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029653077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029667586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735169172286987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029682273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796786963939667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029696747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137409046292305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029711844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13815976679325104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029726481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775962591171265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029741237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382155418395996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029756033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720513880252838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029770636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767804205417633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029785633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732370734214783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029800630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847987353801727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029815581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369217485189438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029830474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819539546966553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029845288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361556500196457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029860287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715818524360657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029860288, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029860288, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029946483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005810856819153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029946484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029946484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029960933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683250546455383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029975413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609425723552704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029989602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746219873428345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030004164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520598411560059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030018810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705433905124664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030033300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435417413711548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030048189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620400428771973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030062905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581609725952148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030077879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379060298204422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030092600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553258776664734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030107461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14594043791294098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030122598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358208805322647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030137142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13441208004951477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030151716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378592848777771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030151717, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030151718, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030239222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007390379905701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030239223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030239223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030253633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573387265205383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030268059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13941703736782074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030282482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645726442337036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030296868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337776929140091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030311559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529525697231293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030326359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390969455242157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030340900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339222490787506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030355812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353009521961212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030370642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347071409225464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030385199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195174932479858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030400041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13176904618740082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030414610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522541522979736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030429323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221579790115356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030444369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389688104391098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030444370, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030444370, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030530931, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010446906089783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030530932, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030530932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030545769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13385866582393646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030560257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351260125637054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030575009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585223257541656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030589513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353573501110077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030604054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375717669725418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030618626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285286843776703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030632818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548016548156738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030647238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945625185966492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030661895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366350531578064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030676837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276223838329315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030691476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13386204838752747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030706327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437041640281677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030721670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353992372751236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030736323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353446125984192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030736324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030736324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030822545, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017223358154297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030822545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030822546, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030837532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271965086460114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030852522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13408982753753662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030867534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355661004781723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030882708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558000326156616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030897589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13933536410331726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030912118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322687149047852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030927917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541437685489655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030943041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343718647956848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030958206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311582803726196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030973616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200053572654724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030988536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357586532831192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031003905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336282640695572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031018857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338139653205872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031033559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097742199897766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031033560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031033560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031120479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028459429740906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031120480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031120481, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031136038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137786865234375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031151032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501277565956116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031166040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354532092809677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031181157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133224844932556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031196063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373054563999176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031211173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13414597511291504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031226416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700738549232483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031241471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398508727550507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031256535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329403966665268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031271553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298399567604065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031286291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277189999818802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031301107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306131958961487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031315998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976138293743134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031330603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12492197751998901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031330604, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031330604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031415825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.504074215888977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031415826, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031415826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031430863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122062385082245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031445720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597489893436432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031460734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133083775639534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031475790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038215041160583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031490161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857069075107574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031505000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12665905058383942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031520064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912385165691376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031534703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222643733024597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031549505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534674048423767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031564426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498313188552856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031578981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880586087703705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031593605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434022665023804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031608376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310667097568512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031623391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291326880455017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031623392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031623392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031708004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5055705904960632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031708005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031708005, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031723061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13057023286819458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031737726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12917742133140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031752038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324013352394104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031767114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12407167255878448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031781956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295383870601654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031796186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12594732642173767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031810521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300540566444397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031825053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12170961499214172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031839142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357160359621048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031853786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728217780590057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031868248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12363668531179428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031883127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311547994613647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031897863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568490207195282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031912438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12886250019073486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031912439, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031912439, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031996358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074251294136047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031996358, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031996359, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785032010564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678714632987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032025741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366099715232849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032040423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582664728164673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032055146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13987931609153748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032069894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365818828344345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032084728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138345867395401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032099511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273384392261505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032114115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607884407043457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032128777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14236125349998474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032143834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409456878900528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032158825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272560954093933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032173631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128570094704628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032188467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280849158763885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032202682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314748227596283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032202683, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032202683, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032286291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096056461334229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032286292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032286292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032301642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14145684242248535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032316648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14235900342464447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032331305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525785505771637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032346090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13951604068279266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032361125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14484068751335144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032375748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.140853613615036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032390783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12598732113838196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032405866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755494356155396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032420980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317688226699829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032436145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13607503473758698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032451111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377616971731186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032465697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330672949552536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032481353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457965850830078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032496794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365053504705429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032496795, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032496796, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032580026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118994116783142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032580027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032580027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032595111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386689841747284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032610683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791711628437042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032626351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175147771835327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032641662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313131034374237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032656919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383269309997559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032672470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398388147354126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032687766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287124037742615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032703087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14129826426506042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032717712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14063970744609833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032732820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531534373760223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032747590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276269257068634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032762049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989234924316406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032776471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13407166302204132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032791133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765942513942719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032791134, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032791134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032871926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5144737958908081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032871927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032871927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032886507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321432739496231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032901034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678477704524994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032915742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335994005203247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032930408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319030076265335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032945152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13400804996490479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032959863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356629014015198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032975015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338094025850296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032989812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327255815267563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033004231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13982409238815308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033019398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318349391222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033034162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891778349876404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033048725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128729164600372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033063450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227564096450806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033078327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593457639217377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033078327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033078327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033161006, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5178228616714478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033161007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033161008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033175923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13349846005439758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033190656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296018749475479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033205510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061143457889557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033220420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304449737071991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033235174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549520075321198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033250439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539214432239532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033265254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539369404315948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033280518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12430291622877121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033295457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311788558959961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033309801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119295239448547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033324771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045194745063782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033339381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333233267068863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033354135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309577077627182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033368735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020072877407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033368736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033368736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033450874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5224289894104004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033450874, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033450875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033465673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378408670425415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033480413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242915272712708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033495128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361626863479614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033509947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039182126522064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033524695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12702296674251556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033539434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320601999759674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033554730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574156165122986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033569846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329241544008255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033584784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313304752111435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033600198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897982239723206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033615354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392767310142517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033629880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753411173820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033645039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201560080051422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033659833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335324823856354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033659834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033659834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033742163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.528555154800415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033742164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033742164, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033757002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13131771981716156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033771769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495539128780365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033786500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797211647033691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033801415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339857757091522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033815811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127409428358078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033830460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955725193023682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033845077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259702742099762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033859683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12410575896501541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033874684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623842597007751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033889608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744521141052246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033904614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457749783992767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033920012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692728638648987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033934962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308896690607071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033949152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235237643122673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033949153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033949153, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785034032820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5375018119812012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785034032821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785034032821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785034047845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237468898296356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034062882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264650106430054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034077876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13385668396949768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034092981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801532983779907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034108010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603994250297546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034122928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315443366765976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034137776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12732350826263428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034152783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350843906402588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034167570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12461381405591965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034182706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304798722267151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034197677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324165314435959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034212799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409270346164703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034228035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14024177193641663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034243034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629133999347687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034243035, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034243035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034324754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5504738092422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034324755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034324755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034340206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136746346950531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034355246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165283203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034369884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13286538422107697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034384773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908126950263977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034400047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299828737974167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034415625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847024738788605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034430505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504919409751892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034445543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13722334802150726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034460482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486097753047943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034475561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082918524742126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034490603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14060133695602417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034505784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474735617637634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034520885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290067583322525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034536084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345292210578918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034536084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034536085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034619036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5654403567314148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034619037, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034619037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034634560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14187422394752502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034649722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255751132965088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034665354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355548679828644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034680465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061444461345673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034695914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546225428581238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034711194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362394273281097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034726550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403567671775818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034741922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557080924510956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034757136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312486082315445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034772722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328807771205902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034788255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328924149274826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034803739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701915740966797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034818865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657203316688538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034834398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739155232906342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034834398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034834399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034917361, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5814656019210815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034917362, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034917362, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034932631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182333111763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034948239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526400923728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034963537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415396213531494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034979412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227973878383636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034994771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371677815914154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035009891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296600103378296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035025225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133645161986351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035039896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240004301071167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035054658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312023550271988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035069311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148286938667297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035084033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125433027744293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035098643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142862915992737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035113383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12888780236244202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035128133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314784437417984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035128133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035128133, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035210565, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5955988168716431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035210566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035210566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035225182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12236940860748291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035240206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270102858543396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035254833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920676171779633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035269722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318559527397156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035284843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131682887673378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035299590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196451425552368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035314757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060317933559418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035330031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320761889219284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035345258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372046172618866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035359854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12813425064086914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035375330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332254260778427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035390614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11818988621234894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035405932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163963913917542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035421132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12536531686782837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035421133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035421134, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035505952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6086992025375366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035505953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035505953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035521115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12168128788471222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035536123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278877556324005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035551355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289099007844925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035566480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237327337265015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035581585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218152865767479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035596795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530179858207703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035611849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982019782066345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035626790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187526375055313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035642102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275699883699417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035657541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771379947662354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035672703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11900296807289124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035688525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13176558911800385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035703989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750427424907684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035719332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912344932556152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035719333, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035719333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035804515, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6209641098976135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035804515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035804516, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035819133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12854643166065216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035834548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11914294213056564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035849909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12242341041564941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035865348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12146605551242828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035880517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12374625355005264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035895700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659117579460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035910859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168239414691925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035926108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12282834947109222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035941032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12572936713695526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035956113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12396722286939621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035971019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580073833465576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035986246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398310750722885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036001261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018551886081696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036016419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12440257519483566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036016420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036016420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036101017, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6342817544937134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036101018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036101018, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785036116319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1169544979929924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036131556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475557088851929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036146619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051199913024902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036161958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209319323301315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036177364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240401491522789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036192343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670642346143723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036206800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203446313738823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036222119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198865413665771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036237213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093774974346161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036252242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371960490942001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036267234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12036240100860596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036282004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12192369997501373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036296896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12463582307100296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036312012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731425672769547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036312013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036312013, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036397457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6630823016166687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036397458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036397458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036412376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976276338100433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036427066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120373085141182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036441906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959010571241379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036456770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322810500860214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036471756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892120003700256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036486802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12269294261932373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036502157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589140444993973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036517698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199980229139328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036532970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032484918832779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036548727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12001372128725052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036564328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300214380025864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036579694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331827193498611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036594703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906418412923813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036609835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657495051622391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036609836, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036609836, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036694445, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7156702280044556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036694446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036694446, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036709358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12012599408626556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036724409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934267729520798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036739621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12281270325183868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036754985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547930747270584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036770164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430314928293228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036785141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167711541056633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036800277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920246481895447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036815286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11829275637865067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036830084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815715044736862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036845056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700911074876785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036859938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476151645183563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036874595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553287506103516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036889322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197122931480408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036904606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12482064217329025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036904606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036904607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036989222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.742989182472229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036989223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036989223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785037004435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384270131587982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037019490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565243452787399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037034429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165732741355896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037049861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12088752537965775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037065035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800241678953171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037080208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149990558624268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037094707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228587478399277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037109846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783343017101288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037124922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113902181386948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037140459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592024564743042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037155710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12137234210968018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037170918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424882709980011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037186378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261860579252243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037201726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110435500741005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037201727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037201727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037286019, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7530915141105652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037286019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037286020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037301432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258150041103363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037316897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251317709684372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037332293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600583046674728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037347846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313408613204956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037363269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11944250017404556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037378932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.117516428232193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037394687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12056851387023926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037410647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11993435770273209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037426113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492573469877243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037442296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233106255531311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037457882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11487557739019394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037473314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146988719701767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037488553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12369818985462189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037504269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389931291341782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037504270, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037504270, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037587632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7565168738365173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037587633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037587633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037602955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11974117159843445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037618451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940973997116089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037633871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152268201112747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037649374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348595470190048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037664793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769302397966385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037680445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312660574913025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037695853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624450236558914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037710902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100059226155281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037726285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944774746894836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037741516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691916197538376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037757175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12271939218044281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037772870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515876859426498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037788394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12026418745517731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037804345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11967593431472778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037804345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037804345, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037887417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7569957971572876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037887417, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037887418, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037902951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813376843929291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037918102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644461005926132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037933330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523988097906113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037949027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471188068389893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037964468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267199575901031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037979965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289730876684189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037995363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11764771491289139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038010635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454169452190399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038025913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259499400854111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038041132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183662176132202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038056123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559078305959702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038071521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180499941110611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038086816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070932447910309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038102044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133958101272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038102082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038102083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038183065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7583979368209839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038183066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038183066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785038198424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412256419658661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038214103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592866688966751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038229708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436646431684494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038245242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882340371608734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038260909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762213170528412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038276655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313130706548691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038292024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113291896879673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038307184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119745671749115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038322459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589708179235458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038337746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036043614149094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038352848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061279684305191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038368066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275241523981094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038383227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707775503396988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038398489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608989745378494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038398490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038398490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038480985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597151398658752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038480985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038480986, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038496136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554756969213486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038511490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347124725580215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038526524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09553229063749313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038541527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432089865207672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038556580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273107677698135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038571497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478318482637405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038586666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08832807093858719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038601801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082974374294281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038617148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111813247203827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038632537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682692170143127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038648139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649792850017548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038663446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052849352359772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038679109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034815967082977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038694628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766758024692535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038694629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038694629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038778227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7616530060768127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038778228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038778228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038793584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11566861718893051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038808840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218993365764618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038823443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297313332557678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038838967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468433052301407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038853813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09950383752584457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038868988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224831104278564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038884056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613743960857391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038899172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704206675291061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038914086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09773766249418259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038929321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090969666838646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038944269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045079156756401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038959275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663264244794846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038974276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843393951654434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038989410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342356353998184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038989410, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038989411, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785039072674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7626348733901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785039072676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785039072676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785039087920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190195381641388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039102781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919563680887222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039118064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202956944704056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039133407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527824610471725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039148902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163237690925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039164388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301141232252121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039179748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002952814102173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039195283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057560071349144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039210762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070713922381401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039225974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655330121517181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039241314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259772628545761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039256043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09414374828338623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039271726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809128940105438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039286948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699663311243057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039286949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039286949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039370260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7646677494049072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039370261, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039370261, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039385308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0924614667892456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039400561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494410991668701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039415743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090574711561203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039431218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711456298828125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039446554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121891438961029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039461886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137281358242035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039477266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09709372371435165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039492569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162955313920975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039508058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578056424856186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039523358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487069934606552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039538762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756123065948486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039553676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512350499629974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039568459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09750806540250778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039583532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726872831583023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039583533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039583533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039668354, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.764612078666687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039668355, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039668355, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039683896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409193485975266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039699394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051609069108963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039714529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054002046585083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039729860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398494452238083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039745335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818893671035767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039760605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996767342090607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039776131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632104426622391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039790857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674769639968872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039806329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817109793424606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039821824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883419215679169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039837208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632376909255981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039853002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468056589365005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039868412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670265346765518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039883867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583961546421051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039883868, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039883868, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039967364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7663647532463074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039967365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039967365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039982617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10141310840845108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039997984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626933515071869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040013070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784891992807388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040028490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122011601924896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040043756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09124260395765305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040059136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992942526936531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040074324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385582596063614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040089717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696138441562653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040104837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726891458034515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040120037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115097463130951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040135780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379917711019516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040151416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032093524932861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040167068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997994244098663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040182552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157351732254028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040182552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040182553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040266615, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659134864807129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040266615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040266616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785040282224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785884410142899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040297758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164423286914825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040313223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136256158351898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040328877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09162995964288712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040344013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502698063850403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040359070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712441802024841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040374897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149239540100098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040390343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245548725128174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040406096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12205622345209122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040421667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464732348918915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040437080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098726093769073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040452603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1225229799747467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040468186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935721009969711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040483841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118056625127792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040483841, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040483842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040567406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7658413052558899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040567407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040567407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040582911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414224117994308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040598279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815498769283295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040613539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992925614118576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040629201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665899515151978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040644711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693682938814163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040660163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281440407037735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040675946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660535842180252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040691696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532119870185852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040707267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664056777954102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040722769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977542608976364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040738591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12530894577503204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040754402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734692960977554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040769828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11761261522769928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040785114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835257917642593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040785115, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040785115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040870806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661381959915161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040870807, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040870807, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040886364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142486110329628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040902273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152065247297287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040918366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117226630449295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040934577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786635220050812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040950667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143484115600586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040966867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11924447864294052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040983154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12704825401306152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040999100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12067095190286636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041014993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667440086603165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041031170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126758903265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041046998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857183486223221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041062963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847475379705429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041078812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529204994440079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041094802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491795629262924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041094803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041094803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041181771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7677146196365356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041181772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041181772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785041197842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963820457458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041213872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770595818758011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041229404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828570276498795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041245505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844992101192474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041261384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090495437383652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041277277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019238829612732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041292924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142561212182045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041308315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799622535705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041323712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060815304517746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041338735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010023415088654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041354237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466131567955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041369304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09709160774946213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041384491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482507199048996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041399617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093088448047638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041399618, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041399618, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041485260, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768900454044342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041485261, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041485262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041500330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912559181451797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041515552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026235967874527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041530807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09685064107179642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041546052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865399241447449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041561450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11548111587762833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041576579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048249825835228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041591718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577867180109024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041607042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466279089450836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041622322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194838792085648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041637687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12002065777778625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041653239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11742866039276123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041668731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108470171689987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041684562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100852832198143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041700052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12080330401659012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041700053, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041700054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041784001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691178917884827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041784002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041784002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041800083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114792674779892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041815745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965057909488678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041831126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489171743392944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041846624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387007892131805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041861729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938257724046707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041877116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531437397003174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041892510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671354085206985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041907796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123092770576477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041923305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489947348833084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041938777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782451927661896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041954048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554792732000351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041969292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152544617652893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041984357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630789399147034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041999613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209311544895172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041999614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041999614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785042082323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697376012802124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785042082324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785042082324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785042097643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000385344028473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042113178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09489002078771591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042128324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09593844413757324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042143584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09669487923383713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042158487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691113978624344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042173769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09378800541162491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042188796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033750534057617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042203869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09740402549505234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042219298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344599187374115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042234684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857328027486801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042250436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479602962732315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042265969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404420644044876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042281519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429620534181595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042297611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135905981063843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042297611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042297612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042381150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7703872323036194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042381151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042381151, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785042396707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09856442362070084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042412288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075512245297432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042428036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996353626251221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042443339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015990749001503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042458656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998466819524765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042473820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687122493982315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042488999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067895591259003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042504358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645583271980286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042520007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027696281671524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042535226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095632076263428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042550605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978649020195007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042565893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024080291390419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042581125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942089557647705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042596330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433225125074387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042596330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042596331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042680302, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702544331550598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042680302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042680303, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042695607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987614840269089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042710946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667978227138519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042726254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0912221297621727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042741537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345527529716492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042757065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643618553876877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042772286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820174962282181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042787534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065710335969925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042803000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387618839740753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042818561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952940046787262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042834164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039847657084465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042849912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026851087808609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042865200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871800780296326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042880912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567653924226761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042896546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892289131879807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042896546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042896547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042979495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711983919143677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042979496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042979496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042995040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281995683908463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043010581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038026511669159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043025869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418958216905594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043040791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290750861167908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043056623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713807493448257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043072007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012154072523117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043087457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09878566116094589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043103071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212603211402893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043118586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201440542936325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043134082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518346726894379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043149442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387296229600906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043164618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241600126028061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043180087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441216826438904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043195337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716909915208817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043195337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043195338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043279720, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712945342063904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043279721, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043279722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785043295085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854844003915787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043310659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197083652019501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043326107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375738888978958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043341596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109477490186691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043357105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445991903543472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043372054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219404637813568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043387370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125410556793213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043402585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075993627309799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043418106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291006207466125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043433871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658794641494751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043449425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312892496585846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043464914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451953113079071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043480687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504025965929031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043496121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920656681060791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043496121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043496122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043580420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718395590782166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043580421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043580421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043595912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142170637845993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043611312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09944124519824982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043626690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678368806838989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043641602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089563578367233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043657302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085607036948204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043672379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008895188570023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043687802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802015662193298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043703363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115870900452137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043718774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11404488235712051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043734023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034260153770447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043749296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393059253692627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043764648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301709920167923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043779948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538244992494583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043795165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739403963088989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043795166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043795166, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043878641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718482613563538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043878641, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043878642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043894018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208604484796524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043909369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037367790937424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043924932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091918796300888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043940223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544835776090622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043955534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0943378359079361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043971248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241808533668518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043986386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014930009841919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044001645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793294757604599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044017187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340537875890732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044032836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361028462648392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044048170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976092517375946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044063704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415565013885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044079245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061646044254303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044094953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294665396213531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044094953, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044094954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044178324, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721467018127441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044178325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044178325, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785044194136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294101923704147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044209739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966439545154572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044225193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095907911658287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044240855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676144063472748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044255983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145042091608047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044270967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785739868879318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044287143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827811598777771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044303067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922399163246155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044318842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621065437793732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044334921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12265443801879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044350693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732336342334747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044366478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029677093029022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044382453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11588132381439209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044398061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101212427020073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044398062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044398062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044481220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717768549919128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044481221, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044481222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785044496925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133842021226883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044512774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957717150449753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044528488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105159670114517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044544376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667456686496735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044560158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445009171962738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044575979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387960612773895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044591711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693113505840302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044607563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471643298864365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044623624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214888095855713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044639603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119716577231884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044655676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587132722139359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044671645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747776716947556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044687523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701770335435867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044703541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889071762561798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044703541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044703541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044787078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721139192581177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044787079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044787079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044803112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11968272924423218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044818910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831928998231888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044834699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920900642871857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044850149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388079077005386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044865726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774396359920502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044881703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546460539102554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044897508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189161986112595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044913606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724760383367538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044929562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322631686925888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044945356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649997740983963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044961057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126059889793396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044976810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545646399259567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044992458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673485487699509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045008373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055169478058815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045008374, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045008374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045092782, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728986740112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045092783, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045092783, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785045108608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009503155946732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045124568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12148874998092651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045140257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982910543680191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045155939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336059123277664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045171691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599365621805191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045187418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925886780023575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045203327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088276207447052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045219278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018630862236023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045235384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263208091259003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045251145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225865036249161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045267127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030004918575287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045283236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095906049013138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045299167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187516897916794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045315030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932359844446182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045315031, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045315031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045400494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732216715812683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045400495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045400495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785045416006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795496195554733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045431759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051807552576065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045447298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027785390615463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045462778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331521928310394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045478305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554412752389908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045494008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147075891494751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045509378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804431885480881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045524815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819686204195023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045540382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861279278993607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045555837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166864842176437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045571448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465064644813538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045586765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494239628314972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045602389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013364046812057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045618074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627517640590668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045618075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045618075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045703307, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737046480178833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045703308, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045703308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045718703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142604798078537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045734211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487498342990875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045749727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041959583759308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045765572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09229335933923721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045781224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416031420230865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045796758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470795631408691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045812642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260246694087982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045828739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250803619623184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045844444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735689103603363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045860552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308591276407242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045876581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095443144440651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045892582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123255267739296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045908724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735747963190079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045924782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714593529701233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045924782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045924783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785046010823, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739094495773315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785046010824, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785046010824, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785046026382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981611162424088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046042115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307557880878448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046057530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051390320062637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046072651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922697395086288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046088807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584432423114777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046104424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990425944328308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046119912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344468802213669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046135819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867152363061905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046151580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843443870544434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046167015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797779262065887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046182593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036918312311172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046198055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993514284491539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046213289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656377673149109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046228446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08717922866344452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046228447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046228447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046313603, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739593386650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046313604, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046313604, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785046328975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.095250204205513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046344351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129730939865112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046359761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753466188907623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046375003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014895811676979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046390542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221555829048157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046405982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09035900980234146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046421207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285398364067078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046436440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09753704071044922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046451737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311660915613174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046467255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641548782587051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046482791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003240197896957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046498267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797853767871857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046513530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620857775211334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046528957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338231176137924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046528958, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046528958, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046613331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741925120353699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046613332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046613333, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785046629060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621137380599976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046644888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546307265758514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046660376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09478022903203964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046675889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021609902381897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046691280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826486557722092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046706357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040194541215897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046721299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934709012508392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046737176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137784481048584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046752654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926609486341476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046768261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973326116800308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046784179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665786266326904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046799956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426990687847137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046815518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084439605474472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046830972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861820727586746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046830972, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046830973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046913498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745561599731445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046913499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046913499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046928878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644318908452988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046944334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405196994543076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046959833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314298421144485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046975584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012508928775787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046991284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695026814937592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047007037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09588398039340973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047023324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503676533699036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047038965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339133441448212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047054292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007937788963318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047070198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09890183806419373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047085532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09765660762786865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047100911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233468562364578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047116559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09087993949651718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047132184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072629913687706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047132185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047132185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047216305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748035788536072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047216307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047216307, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785047232098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924429655075073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047247977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121461749076843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047263457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0986662358045578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047279427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941512137651443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047295210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145904123783112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047311153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288138687610626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047326691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235581547021866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047342232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527245700359344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047358067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142175108194351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047373225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274166405200958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047388842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010979413986206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047404773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830564796924591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047420318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580617189407349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047436067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611627250909805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047436067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047436068, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047520237, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749396562576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047520238, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047520239, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785047535798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621857643127441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047551445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09854690730571747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047566909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691629350185394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047582699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651107877492905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047598343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228796303272247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047613793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767363011837006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047629309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09720925986766815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047645119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834668785333633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047660786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572017729282379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047676527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050526350736618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047692303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224377363920212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047708175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982493311166763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047723902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666386038064957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047739705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992799699306488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047739705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047739706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047824949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752670645713806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047824950, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047824950, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047840523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326206684112549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047856098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568917542695999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047871775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09432324767112732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047887725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09890106320381165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047903314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273885726928711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047919341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997005015611649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047935159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665712505578995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047951119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026348173618317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047966943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886025428771973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047982835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928399324417114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047998799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287144780158997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048014204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09859870374202728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048030175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218224465847015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048045594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099744588136673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048045595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048045595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048129492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755571603775024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048129493, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048129493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785048144550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588846355676651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048160661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488937050104141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048176163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607078671455383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048191882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132410913705826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048207790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638770461082458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048223547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791976004838943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048238961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720154643058777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048254546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0978962630033493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048270234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620579868555069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048285563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330219566822052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048301105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844404995441437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048316296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340584069490433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048332033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064678430557251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048347492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09575255215167999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048347532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048347532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048431070, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754956483840942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048431071, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048431071, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785048446444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934557229280472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048461737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528886318206787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048477153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207775235176086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048492589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445945709943771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048507901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812358558177948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048522934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312372028827667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048538228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810960829257965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048553676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897647053003311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048569295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114044189453125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048584797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121302843093872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048600573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476586759090424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048616244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983632281422615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048632089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658685117959976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048647912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458948463201523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048647950, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048647950, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048732347, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759877443313599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048732348, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048732348, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785048747908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979757010936737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048763472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09727533906698227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048778980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127978190779686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048794765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795223504304886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048810206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405187100172043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048825259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192343384027481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048841279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740824788808823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048857106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058249115943909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048872903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456175357103348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048889038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082591712474823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048905005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155124008655548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048920734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025093495845795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048936537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361686140298843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048952236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119285225868225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048952236, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048952236, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785049035418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776131808757782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785049035419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785049035419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785049051281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773396492004395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049066907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012009531259537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049082921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389598250389099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049098800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049704998731613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049114567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825876891613007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049130546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797278583049774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049146522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101422980427742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049162290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216869205236435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049178148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12057866901159286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049194020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983136296272278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049209725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088915467262268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049225703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425311326980591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049241853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366291552782059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049257847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849743336439133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049257848, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049257848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049339590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762653827667236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049339592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049339592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785049355496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427692323923111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049371558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832380294799805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049387678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309368163347244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049403821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252378672361374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049419946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072394996881485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049435765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12066096812486649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049451678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175346374511719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049467063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896798223257065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049483406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021643131971359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049499145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903962165117264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049514909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011140793561935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049531137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109600730240345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049547085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399632155895233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049562914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016716808080673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049562914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049562915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049645674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764533162117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049645675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049645675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785049661439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262299120426178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049677346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907259166240692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049693387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815557092428207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049709323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208250373601913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049725354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131477952003479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049741624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056549847126007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049757948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141301840543747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049774403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344520002603531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049790515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610675066709518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049806559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802905261516571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049822861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12932397425174713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049838928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273171752691269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049855460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182940751314163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049871918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163337305188179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049871919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049871919, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049954881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767407298088074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049954882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049954882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049971008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09568304568529129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049987314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148159205913544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050003773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1221008449792862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050020220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639037936925888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050036730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976195335388184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050053056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717987269163132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050069265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197929829359055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050085157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330242455005646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050101299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306557267904282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050116630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11797911673784256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050132197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691560059785843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050147956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095692217350006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050163842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09242291748523712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050179359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338521003723145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050179359, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050179359, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050262893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769556641578674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050262893, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050262894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785050278570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104546695947647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050294149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09669612348079681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050309539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587943881750107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050325030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073482558131218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050340434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956810414791107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050356206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387235134840012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050371817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755886137485504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050387510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266158521175385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050403123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09944037348031998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050418879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09590284526348114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050434544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717573970556259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050450045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677547961473465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050465435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029542088508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050481043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055892914533615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050481044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050481045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050564021, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772437334060669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050564022, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050564023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785050579612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386199504137039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050595540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240408778190613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050611353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404208302497864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050627176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071949377655983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050643030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019618883728981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050659171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994638055562973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050675405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911529511213303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050691268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527073591947556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050707178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063825711607933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050722978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523383319377899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050738926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040949746966362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050754344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853537917137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050769488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534870624542236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050785323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230865865945816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050785361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050785362, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050868966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773850560188293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050868968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050868968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050884564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782922059297562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050900233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150781810283661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050915942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106721431016922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050931836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721345245838165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050947342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041217505931854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050962991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228119790554047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050978607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09591623395681381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050994046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774047672748566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051009372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961051493883133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051024861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024140864610672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051040618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150760412216187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051056108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286574810743332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051071657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09927225112915039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051087314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324124991893768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051087356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051087356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051171055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776333689689636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051171057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051171058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785051186540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09715550392866135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051202065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718408226966858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051217776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0987834706902504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051233268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260947793722153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051248647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09376011788845062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051264302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659845173358917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051279990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352923721075058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051295543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380154848098755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051311656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510361194610596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051327320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197260767221451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051343237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09644870460033417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051359162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094565689563751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051375012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174077540636063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051390503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197876393795013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051390542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051390542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051475037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778106331825256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051475038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051475038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785051490565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098127581179142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051506519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637486726045609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051522343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118655860424042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051537509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633955895900726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051553323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043023526668549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051568952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924318432807922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051584640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023977607488632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051600420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948889911174774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051616466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871562361717224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051632496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918684303760529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051648396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969040006399155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051664371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09458836913108826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051680002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566080570220947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051695609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588618367910385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051695609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051695610, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051781569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778113603591919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051781571, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051781571, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785051797266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315796732902527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051812984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348892748355865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051828908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009859502315521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051844797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046040803194046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051860596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059172451496124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051876135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008690595626831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051891928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907064378261566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051907583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120120644569397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051923392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784300416707993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051939189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133907943964005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051954938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09704776108264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051970972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116553470492363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051987094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11915604770183563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052003058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950537770986557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052003058, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052003059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052088310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781916856765747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052088310, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052088311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785052104469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581424623727798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052120396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090004652738571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052136764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989817440509796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052152732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10088163614273071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052168684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494339466094971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052184749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916020929813385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052200622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047870129346848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052216741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562727600336075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052232286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590075701475143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052247679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09388544410467148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052263999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420989990234375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052279858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022205576300621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052295653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09330407530069351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052311558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993059188127518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052311558, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052311558, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052395334, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783814668655396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052395335, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052395335, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785052411447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062396317720413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052427438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304132640361786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052443076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080648213624954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052458877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065312922000885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052474713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300885140895844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052490231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025804802775383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052505932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610619187355042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052521482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817042738199234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052537234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09806328266859055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052552877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09716376662254333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052568639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045565456151962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052584104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09838316589593887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052599905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184745490550995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052615355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296864807605743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052615355, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052615356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052698165, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786149978637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052698166, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052698166, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785052713849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172569006681442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052729526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784998536109924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052745050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160578578710556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052761118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09362984448671341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052777139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360120236873627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052793146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770443081855774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052809097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595381796360016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052824793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080304741859436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052840734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521402955055237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052856930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924172401428223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052872980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09702498465776443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052888830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766198486089706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052904865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812960773706436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052920585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494979470968246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052920586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052920586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785053004675, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788171768188477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785053004676, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785053004676, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785053020770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025577783584595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053036264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494038462638855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053051365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09216279536485672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053067392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010656714439392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053083117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082434207201004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053098602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110938660800457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053114418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09611161798238754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053130130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426657646894455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053145821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060277447104454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053161467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898915886878967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053177129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380960255861282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053192808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460717231035233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053208362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103017628192902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053224065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466465890407562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053224065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053224066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053307966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790400981903076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053307967, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053307967, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785053323501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094338446855545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053339309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472080111503601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053354901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320564359426498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053370609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087474450469017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053386144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280735045671463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053401836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971592515707016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053417468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308779031038284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053432975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221464723348618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053448466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080191493034363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053463997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09507118910551071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053479538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439208894968033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053495187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11226298660039902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053510973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206388682126999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053526564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821974813938141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053526564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053526565, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053611378, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792752385139465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053611379, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053611379, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785053627254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079661756753922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053642992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09697402268648148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053658799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529815405607224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053674868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282443255186081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053690795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724307388067245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053706796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09552142024040222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053722992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576630383729935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053738895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014472246170044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053754952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105352103710175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053770791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411956906318665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053786181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641738772392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053802581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976242274045944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053818453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931863099336624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053834588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498884111642838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053834588, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053834589, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053918624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791860103607178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053918625, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053918625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785053934898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217022687196732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053951198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073897778987885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053967338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682467371225357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053983548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12002801895141602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053999684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156967356801033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054015969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019643098115921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054032320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11920247226953506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054048396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338551342487335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054064618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056523248553276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054080961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654320567846298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054097288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071474701166153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054113449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054617315530777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054129853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065729483962059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054145975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11548532545566559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054145976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054145977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054230211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796122431755066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054230212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054230212, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785054246628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222530901432037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054262730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433676093816757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054279100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240554064512253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054295404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670425534248352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054311807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833334177732468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054327940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568086057901382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054344457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502792149782181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054360727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355916410684586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054377353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11875844746828079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054393690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527527332305908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785054410066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618325859308243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054426224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123441681265831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054442032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760442167520523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054458410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11602023988962173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054458411, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054458411, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054542168, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794124484062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054542169, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054542169, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785054558043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012333557009697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054574225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106337457895279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054590345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11512844264507294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054606403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110724717378616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054622976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09913923591375351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054639449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076960563659668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054655728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897501021623611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054671971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745072364807129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054688292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004454493522644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054704406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10514990240335464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054720474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447048932313919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054736706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051114648580551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054752791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170361265540123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054769176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11295599490404129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054769177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054769178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054853400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796792984008789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054853401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054853401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785054869489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242217570543289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054885813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625597089529037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054901984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105893552303314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054918296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167774349451065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054934495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210576444864273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054950783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659279465675354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054967200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976205974817276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054983404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798609256744385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054999883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764966160058975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055016209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085800975561142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055032573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11846086382865906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055048761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124740019440651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055064997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042240783572197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055081238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084605008363724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055081239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055081239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055166226, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798551917076111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055166227, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055166227, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785055182089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508094727993011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055198186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103251501917839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055213988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1190112978219986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055230230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627656430006027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055245991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578213632106781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055262139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163464397192001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055278720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652968287467957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055294577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041556298732758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055310386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134968176484108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055325974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944184884428978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055341849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711808294057846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055357568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093656927347183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055373019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09825185686349869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055388686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081087589263916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055388686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055388687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055474256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796524167060852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055474259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055474259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785055489915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224390029907227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055505705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09241347759962082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055521607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09721206873655319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055537338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861765593290329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055553085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794254392385483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055568801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224592477083206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055584304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554806351661682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055600027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623259842395782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055615717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656260699033737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055631245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058173328638077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055646867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823901951313019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055662937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991285964846611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055678863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091062426567078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055694569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826585650444031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055694569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055694570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055780312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798555493354797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055780313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055780313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785055796305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574164986610413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055812110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09575555473566055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055828104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471934080123901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055844256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09649120271205902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055860275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253801941871643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055876114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189506947994232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055892002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689497739076614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055907623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09519552439451218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055923313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527260392904282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055938958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426713526248932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055954115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331381857395172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055970066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100717693567276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055985517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102350153028965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056001237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174252837896347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056001238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056001238, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056086079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798238396644592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056086081, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056086081, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785056101941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068833827972412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056117746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912340342998505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056133864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019151508808136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056149503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296670347452164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056165477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889953583478928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056181143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10114128887653351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056196832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09642529487609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056212503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916957467794418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056228542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364054888486862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056244517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818169265985489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056260372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358307510614395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056276074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749109834432602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056291893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014968678355217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056307383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661363810300827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056307383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056307384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056390174, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800012826919556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056390175, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056390175, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785056406002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09455364942550659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056421475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09933817386627197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056436958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738662838935852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056452469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975464642047882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056467929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403042286634445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056483928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0957406535744667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056499468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182668268680573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056515067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09813594818115234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056530526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788218140602112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056546250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457142442464828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056561973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09304811805486679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056578109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806923359632492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed1026364325.log b/recommendation_v4/rcp_logs/gbs_8192/seed1026364325.log new file mode 100644 index 000000000..7af274146 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed1026364325.log @@ -0,0 +1,639 @@ +:::MLLOG {"namespace": "", "time_ms": 1784941524458, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784941524458, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784941546250, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "seed", "value": 1026364325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784941546252, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784941546253, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784941546253, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784942465782, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784942465783, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784942466213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784942596928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742241263389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784942609239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938042521476746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942621763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886286318302155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942634396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392468363046646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942647277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389351785182953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942660056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904841244220734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942672988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391369104385376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942686034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860578835010529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942699163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389235556125641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942712140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910414278507233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942725520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387670785188675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942738733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864858448505402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942751991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13891690969467163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942765130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906390964984894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942765131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942765131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942904084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000073313713074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942904085, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942904085, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942916912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908334076404572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942930103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897421956062317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942943444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897790014743805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942956242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888296484947205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942969452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910463452339172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942983282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899990916252136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942996576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909557461738586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943010011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893473148345947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943023640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388172060251236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943037101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894455134868622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943050901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878437876701355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943064640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867396116256714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943078236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878978788852692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943092845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386394053697586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943092846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943092846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943185796, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002493858337402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943185796, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943185797, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784943199219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385965198278427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943212859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864897191524506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943230233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848280906677246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943243821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819153606891632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943257476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839076459407806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943270926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847528398036957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943284523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817468285560608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943298172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792595267295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943311670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377793252468109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943325059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137987419962883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943338677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808971643447876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943352174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822215795516968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943366277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379290223121643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943380236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809587061405182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943380237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943380237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943475051, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005601644515991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943475052, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943475053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784943488658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756555318832397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943502565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790832459926605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943516477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744895160198212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943530806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376878172159195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943544682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378932148218155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943558782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794592022895813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943572908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719408214092255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943586844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769276440143585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943601306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374053955078125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943615643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863179087638855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943629762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685248792171478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943644002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379208117723465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943658107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598769903182983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943672280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695167005062103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943672280, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943672281, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943766988, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5010226368904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943766989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943766990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943780375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658630847930908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943794181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593986630439758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943807758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375114619731903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943821490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523073494434357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943835580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682076334953308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943849373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431355357170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943863455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13587836921215057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943877394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580718636512756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943891589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793492317199707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943905701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572774827480316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943920089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1461067497730255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943934341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586664199829102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943948429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134622260928154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943962560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771849870681763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943962561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943962561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784944058716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501638650894165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784944058717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784944058717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784944072379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356346756219864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944086208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13944940268993378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944099926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361686885356903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944113758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356630504131317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944127647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514244556427002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944141718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898661732673645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944155612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374638557434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944169501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359376788139343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944183948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376428186893463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944197824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200019299983978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944212505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179898262023926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944226326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351270079612732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944240396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244356215000153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944255262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870859146118164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944255262, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944255262, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944350243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024182200431824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944350244, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944350244, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784944364252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410784304141998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944378484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369950652122498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944392565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588041067123413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944406415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497388362884521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944420387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375962346792221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944434449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12844812870025635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944448132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565988838672638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944462131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297820806503296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944476201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350965082645416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944490733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13272623717784882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944504962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369277119636536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944520111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474500179290771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944535134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352992057800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944549585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359159231185913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944549586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944549586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944644439, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034306049346924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944644440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944644440, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784944659363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281022012233734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944673831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13385939598083496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944688626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551747798919678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944703732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355067640542984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944718096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392674446105957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944732582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333032101392746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944747703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353289783000946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944762424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356488943099976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944776597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312554359436035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944791335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13200685381889343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944805889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567785918712616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944820689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394410908222198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944835296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336537778377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944849642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13066062331199646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944849642, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944849643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944936301, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046725273132324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944936302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944936302, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944951641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376209855079651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944966528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486310839653015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944981225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354445070028305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944996365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150626420974731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945011166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356754183769226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945025646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13408879935741425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945040813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714149594306946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945055518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391271233558655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784945070261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292187452316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945084532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974891066551208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945098626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12774540483951569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945113228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301239907741547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945127782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951353192329407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945141980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12486007809638977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945141981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945141982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945236966, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.506056547164917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945236968, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945236968, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784945251538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311056762933731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945265725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357857882976532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945280578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331709772348404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945295183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303158402442932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945309298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808731198310852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945324946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267431080341339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945339920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12904247641563416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945354060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323758363723755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945368793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549837470054626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945383549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498108297586441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945397794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12865203619003296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945412229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434208929538727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945426675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310819536447525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945441438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939304113388062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945441439, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945441439, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945537323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5077449679374695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945537324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945537324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784945552235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303529590368271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945566438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292913854122162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945580331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317173719406128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945595080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1238841786980629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945609972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12950190901756287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945624440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12572915852069855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945638809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003243505954742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945653581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198100239038467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945667744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588206470012665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945682666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13707734644412994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945697394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12353332340717316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945712490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231696754693985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945727302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12566415965557098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945742024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894652783870697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945742024, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945742025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945835358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095719695091248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945835359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945835360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945849380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652929663658142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945864761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629518449306488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945879263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568352162837982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945893730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398308128118515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945907938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655361533164978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945922313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829608261585236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945936605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736651301383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945950969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12620896100997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945965564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14217360317707062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945980189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410769522190094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945994923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12739962339401245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946009472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283000409603119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946023925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273347914218903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946037798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13131961226463318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946037798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946037799, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946124348, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5115414261817932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946124349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946124350, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784946139227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141310915350914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946154544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14236481487751007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946169148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353446990251541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946183777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13942871987819672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946199114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14488379657268524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946213496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14091186225414276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946228544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12586991488933563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946243770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751104474067688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946258598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181030750274658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946273813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623905181884766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946288715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773547112941742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946304124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330055594444275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946319490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345403641462326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946334617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641314208507538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946334619, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946334620, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946429410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5135456323623657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946429411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946429412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784946443823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865207135677338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946458533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816285133361816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946473553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187214732170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946488346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297539949417114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946503182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377198576927185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946518415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13988132774829865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946533369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329226791858673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946548716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14115020632743835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946563189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1407945603132248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946577992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509155809879303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946593145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277146190404892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946607828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991692125797272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946622313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432198762893677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946637474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780366837978363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946637475, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946637475, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946732892, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160844922065735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946732894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946732894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946747414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232871890068054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946761872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697032630443573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946776972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350440561771393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946791197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187287747859955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946806122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13400965929031372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946821008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334901630878448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946836083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382023572921753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946851348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326749622821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946865671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13972164690494537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946880739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184455037117004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946895686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12878911197185516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946910045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310272514820099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946924655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221269845962524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946939671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597051799297333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946939671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946939671, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784947035664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.519473135471344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784947035665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784947035665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784947050523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361412286758423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947065505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12315766513347626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947080553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13062527775764465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947095418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031430542469025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947110473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353178322315216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947125594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552208244800568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947140469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13547323644161224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947155897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1240696832537651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947170735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114574551582336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947185194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310739517211914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947199893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032008707523346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947214681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319939374923706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947229284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308547854423523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947243823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303940862417221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947243824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947243825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947339427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5238083004951477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947339428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947339428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784947354585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779179751873016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947369112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241977989673615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947384091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354238867759705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947398935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034698367118835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947413572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719006836414337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947428421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13207495212554932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947443349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356399655342102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947458604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288187980651855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947473915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129909336566925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947489065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899220526218414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947504223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13379517197608948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947518536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769134879112244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947533549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13189508020877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947548427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333012580871582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947548428, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947548430, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947644480, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5296273827552795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947644481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947644482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947659135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133026659488678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947673756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350405365228653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947688766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377696841955185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947703460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340483605861664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947717982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751592695713043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947732782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12940648198127747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947747401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12590952217578888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947761890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239841878414154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947776803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606929779052734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947791820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733412325382233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947806905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471245765686035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947822215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692523002624512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947837172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13095711171627045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947851514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12343057990074158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947851515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947851515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947943537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.537490725517273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947943538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947943538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947958765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252437114715576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947973910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13267388939857483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947989129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346615433692932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948004512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781213760375977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948019708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642406463623047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948034680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142657279968262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948049920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12737233936786652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948064948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503198325634003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948079879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12465999275445938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948095355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303391307592392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948110525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256458938121796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948126097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444629311561584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948141592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14036735892295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948156632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622508943080902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948156633, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948156633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948252745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5491786003112793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948252746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948252747, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784948268508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659654557704926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948283525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157573342323303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948298165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13272322714328766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948313107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894827663898468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948328393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300020068883896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948343868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855160772800446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948358802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350577473640442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948373708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736079633235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948388933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348874568939209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948404210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13085348904132843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948419237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14068524539470673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948434585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476234674453735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948449709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922582030296326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948465174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312572240829468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948465175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948465175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948559544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5625792145729065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948559545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948559545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784948575354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14216236770153046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948590656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271337747573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948606651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359954953193665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948622332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097642362117767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948637957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550373911857605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948653093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356338441371918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948668561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340518444776535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948683941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571935892105103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948699379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119418919086456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948714978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302385807037354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948730454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300170004367828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948746297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136993870139122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948761574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634224236011505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948777313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754256069660187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948777314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948777314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948873107, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5775254368782043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948873109, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948873109, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948888627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178801536560059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948904023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545621931552887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948919701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424231112003326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948935698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204637169837952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948950766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733908534049988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948965762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952473759651184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948981175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391610980033875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948996213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12426531314849854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949011404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124793767929077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949026096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181839883327484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949041411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131117582321167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949056520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167811930179596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949071799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943990528583527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949087368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098227977752686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949087368, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949087369, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949183663, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5897506475448608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949183665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949183665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784949198425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12252422422170639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949213929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711501121520996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949229242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969499826431274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949244502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332955539226532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949260231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221696019172668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949275414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12219476699829102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949291005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306719183921814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949306493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225732743740082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949321494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704702258110046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949335997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12816287577152252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949351383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364015519618988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949366426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11809934675693512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949381619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12171386927366257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949396707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257641315460205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949396708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949396708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949492754, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6016207337379456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949492755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949492755, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784949508018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12178269028663635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949523106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279434859752655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949538514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285974383354187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949553908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13231416046619415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949568864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12194996327161789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949584307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524633646011353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949599301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302611380815506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949614194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12192552536725998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949629657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12701445817947388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949645182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760303914546967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949660664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11876402050256729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949676637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318582147359848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949692144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275564432144165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949707593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903186678886414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949707593, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949707594, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949803698, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6113557815551758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949803699, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949803699, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949818167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286386251449585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949833568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11937104165554047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949848921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12247850745916367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949864198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12202344089746475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949879252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12399055808782578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949894519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12643317878246307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949909383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671560257673264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949924537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12305118143558502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949939781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12597112357616425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949954664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408066540956497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949969801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592482775449753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949984944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12464570999145508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949999873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083588749170303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950015420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498640269041061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950015420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950015420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950111917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.619658350944519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950111918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950111918, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784950127263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11749959737062454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950142646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500047147274017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950158002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997514009475708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950173558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196364998817444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950188840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418511509895325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950203921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670851707458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950218283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12029951810836792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950233528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12243445217609406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950248923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12153959274291992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950264173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394033581018448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950279519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218964084982872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950294860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12226888537406921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950309968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12539397180080414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950325335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11713549494743347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950325336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950325336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950420981, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.630948007106781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950420982, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950420982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784950436524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12011384963989258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950451642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251520365476608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950466922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032222956418991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950482250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392379552125931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950497324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934105515480042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950512788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12241443246603012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950528240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632782965898514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950543947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12119251489639282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950559497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034821942448616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950575109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957269161939621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950590902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296570301055908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950606719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386486142873764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950621937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051925271749496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950636968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172223687171936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950636969, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950636969, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950733178, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6497419476509094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950733180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950733180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950748039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210394874215126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950763231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975222826004028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950779059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12400878220796585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950794179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657002031803131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950809653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148744523525238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950825391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11794503033161163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950840658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977992951869965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950856037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919103562831879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950871314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979210585355759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950886378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819546669721603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950901868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573594063520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950916633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757631599903107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950931532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330569535493851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950947388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723281979560852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950947388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950947389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784951043343, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6745119094848633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784951043344, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784951043344, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784951058650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12514157593250275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951074155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11718439310789108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951089598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11683519184589386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951105190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299320101737976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951120859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087450385093689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951136215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217683553695679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951150547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414947360754013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951166009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909619182348251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951181830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112120121717453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951197750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717221885919571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951213572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12420021742582321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951229498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11588915437459946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951245196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796732783317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951261170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348115652799606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951261171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951261171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951356227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7271044850349426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951356228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951356228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784951372141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12652361392974854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951387869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449965089559555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951403907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831612139940262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951419720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366499215364456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951435389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049419432878494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951451625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925993114709854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951467553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12167107313871384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951483663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12161265313625336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951499814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11572716385126114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951515982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314544826745987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951532016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547024548053741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951547215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11668617278337479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951562435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12501144409179688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951578448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662144958972931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951578449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951578449, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951674861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7501415014266968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951674862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951674862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951690728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12292882055044174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951707109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11950357258319855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951722749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414574086666107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951738912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151285395026207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951754881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950080305337906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951770665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537306755781174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951786703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799280345439911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951802114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179741472005844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951817829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102556586265564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951833908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090901717543602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951849920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12332583218812943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951865901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107154481112957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951881852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12077640742063522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951897927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940683424472809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951897927, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951897927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951994276, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597251534461975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951994277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951994277, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784952010582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11943729221820831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952025847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11867070943117142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952041259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163618192076683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952057114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671170592308044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952072724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324077308177948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952088762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435380041599274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952104212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11993025243282318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952119844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515036970376968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952135528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372337698936462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952150969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297408699989319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952166465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071285530924797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952182112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243951320648193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952197428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101038753986359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952213340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491339653730392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952213377, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952213377, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952308227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7645624876022339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952308228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952308228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784952323757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432390868663788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952339925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699755698442459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952356209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11534254997968674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952371809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057397723197937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952387850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768117755651474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952404025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445451527833939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952419728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313016712665558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952435232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267698556184769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952450410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623125523328781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952466071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153814196586609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952481452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092351585626602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784952497223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342328786849976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed138251694.log b/recommendation_v4/rcp_logs/gbs_8192/seed138251694.log new file mode 100644 index 000000000..0939b8339 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed138251694.log @@ -0,0 +1,681 @@ +:::MLLOG {"namespace": "", "time_ms": 1784940678141, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784940678141, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784940699724, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "seed", "value": 138251694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784940699726, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784940699732, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784940699732, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784941584276, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784941584277, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784941584733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784941737948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866974413394928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784941750230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830728828907013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941763074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849303126335144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941776198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847102224826813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941789514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856860995292664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941802827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847239315509796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941816237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382628083229065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941829655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868878781795502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941843175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876457512378693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941856619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853897154331207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941870440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847948610782623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941884012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388532519340515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941897946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383674293756485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941911568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907232880592346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941911568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784941911569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784941996629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013336539268494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784941996630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784941996630, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942009761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839007914066315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942023058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850615918636322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942037490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383703351020813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942051110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834752142429352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942065010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138162761926651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942079381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380109041929245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942092654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138204887509346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942106289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812881708145142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942120851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791970908641815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942134663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827012479305267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942148899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385221630334854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942162917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853959739208221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942176670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802139461040497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942191015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879024982452393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942191016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942191016, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942272932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5016124248504639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942272933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942272933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942286821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768979907035828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942300632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828414678573608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942314516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373881846666336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942328588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365291178226471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942342250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776709139347076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942355836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801166415214539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942369507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741536438465118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942384037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698731362819672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942398234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650307059288025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942412287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372741460800171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942426481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372509002685547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942440362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375759094953537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942454768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378057599067688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942468359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762608170509338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942468360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942468360, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942549323, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019965767860413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942549324, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942549324, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942562779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367434412240982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942576547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373411864042282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942590259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650484383106232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942604869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719111680984497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942618934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728244602680206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942633205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373216211795807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942647270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651223480701447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942661255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730812072753906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942675743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681188225746155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942689971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386224925518036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942703988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13616842031478882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942717968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799263536930084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942731912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497263193130493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942746295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643650710582733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942746296, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942746296, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942830866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5024652481079102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942830867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942830868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942844560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609302043914795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942858304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531087338924408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942871840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744038343429565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942885631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432861864566803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942899551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364355832338333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942913207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332699954509735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942927351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523021340370178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942941115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519413769245148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942955045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781386613845825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942968849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493362069129944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942982690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14763686060905457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942996926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354249268770218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943010640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389833271503448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943024459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753929734230042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943024460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943024460, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943107290, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503047525882721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943107291, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943107291, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943121020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350109726190567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943134737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970838487148285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943148571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584090769290924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943162301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282990455627441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943176165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347324252128601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943190211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392994523048401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943204108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331050544977188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943218340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277648389339447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943232294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289564847946167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943245869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114900887012482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943259759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087984919548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943273243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471412658691406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943286951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125014305114746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943301112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911239802837372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943301113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943301113, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943383125, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036991834640503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943383126, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943383126, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943397030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331815600395203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943410750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321655988693237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943425034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356871873140335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943438982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478785753250122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943453067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773882389068604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943467155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733687460422516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943480811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135287344455719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943494705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290522664785385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943508766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333162784576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943523177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234490156173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943537210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316906988620758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943551575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430175185203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943566461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530348241329193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943580710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329903781414032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943580711, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943580712, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943663040, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046447515487671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943663041, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943663041, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943677416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240981101989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943691554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372398912906647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943705779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355431228876114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943720234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557332754135132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943734499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396748572587967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943748520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302500545978546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943763351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524845242500305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943777374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332775205373764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943791397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330786645412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943805474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170461356639862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943819344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13570375740528107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943833664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361454010009766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943847555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311755657196045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943861303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050971925258636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943861303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943861304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943944079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058592557907104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943944080, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943944081, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784943958822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806255161762238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943973182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346038281917572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943987575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541436195373535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944002138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13120798766613007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944016317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353268802165985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944030761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340671330690384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944045371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370886117219925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944059857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341196894645691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944074224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277165591716766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944088242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961351871490479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944101807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12749366462230682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944115630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132957324385643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944129620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946678698062897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944143246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1244872659444809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944143246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944143247, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944223960, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070699453353882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944223961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944223961, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944237928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102023303508759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944251667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583701848983765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944265676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310423493385315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944279725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025155663490295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944293463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1280190646648407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944307723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12647002935409546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944322250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879173457622528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944336290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233129680156708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944350534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506591320037842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944364779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248164251446724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944378710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285477727651596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944392632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433130085468292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944406876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112208247184753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944421401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12914060056209564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944421402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944421402, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944503601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086745023727417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944503602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944503602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944518065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053375482559204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944532253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931300699710846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944546045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331900805234909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944560435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12387964874505997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944574633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926073372364044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944588245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256505697965622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944601944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299981325864792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944615878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216902956366539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944629531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584913313388824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944643586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705600798130035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944657431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236468032002449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944671832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231631264090538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944686023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12549695372581482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944700139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289748251438141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944700139, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944700139, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944783730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5105555057525635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944783731, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944783731, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784944797443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13657806813716888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944812100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643555343151093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944826280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572748005390167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944840288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14000573754310608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944854251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677887618541718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944868329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840492069721222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944882228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12735222280025482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944896371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261424720287323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944910553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14225342869758606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944925159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14094416797161102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944939678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718594074249268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944954041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830393016338348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944968316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13272882997989655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944982062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13135287165641785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944982063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784944982063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945066630, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125978589057922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945066631, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945066631, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945081401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14137797057628632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945095893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14259056746959686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945110097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353149563074112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945124421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13934427499771118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945138981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14458467066287994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945153065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1406998336315155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945167542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12576410174369812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945182018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375160813331604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945196508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13174191117286682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945211149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615646958351135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945225624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762350380420685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945239917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284137845039368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945254897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467365503311157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945269758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658714294433594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945269759, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945269759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945353576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5148406028747559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945353577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945353577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945367893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384589821100235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945382698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809290528297424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945397471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156390190124512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945412103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328871250152588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945426728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372330367565155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945441648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395696997642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945456435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325719654560089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945471253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411779224872589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945485425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1407012641429901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945500000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517312705516815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945514218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760290503501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945528045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009122014045715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945541865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383178412914276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945555863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127670556306839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945555863, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945555864, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945641513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5176620483398438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945641514, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945641514, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945655267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205267488956451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945669007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674403727054596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945683011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360807299613953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945697093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317916065454483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945711142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341092437505722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945725198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333257555961609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945739586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377153873443604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945753683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246367871761322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945767595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13973018527030945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945782441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191673159599304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945796726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880949676036835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945810671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311231553554535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945824844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324755698442459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945839067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573423027992249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945839068, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945839068, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945924775, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5217011570930481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945924776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945924776, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784945939154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334647834300995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945953357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228133961558342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945967570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052617013454437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945981778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032186031341553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945995776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478568196296692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946010390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540324568748474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946024522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352274864912033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946039083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12417808175086975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946053193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311016082763672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946067069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309293955564499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946081752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010276854038239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946095721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305039703845978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946109720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13074073195457458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946123496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300451159477234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946123497, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946123497, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946208212, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5270775556564331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946208213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946208213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946222200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769391179084778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946236291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320735514163971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946250273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333255022764206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946264325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13042645156383514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946278336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12685903906822205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946292237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179096579551697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946306645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355409473180771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946320959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326817125082016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946335155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090728223323822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946349681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388615369796753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946364101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365936279296875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946378049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727208971977234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946392816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317090541124344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946406922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308702409267426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946406922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946406923, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946489509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5344972014427185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946489510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946489510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946503590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308101862668991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946517561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13476932048797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946531662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753171265125275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946545783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270117342472076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946559980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12724114954471588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946574063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912632524967194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946588072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560974061489105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946602294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12377482652664185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946616911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574884831905365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946631446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698239624500275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946645762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342490017414093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946660724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12684254348278046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946675281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075877726078033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946689266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231008842587471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946689266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946689267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946774385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5446968078613281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946774386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946774386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784946789030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320635825395584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946803440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230593502521515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946817865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333594173192978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946832310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137422576546669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946846698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588282465934753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946861102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113069534301758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946875427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126657634973526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946889794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469095528125763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946904018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12429992854595184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946918564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987950444221497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946932998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318736970424652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946947622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335400491952896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946962315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398126780986786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946976706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357780247926712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946976707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784946976707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947060278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5587624907493591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947060279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947060279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947075177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643589615821838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947089552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097447156906128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947103658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320662647485733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947118191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385875642299652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947133161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293112337589264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947148379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779115676879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947163065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430531322956085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947177895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663138449192047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947192745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392706215381622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947207716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004374504089355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947222685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968510925769806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947237851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399049639701843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947252971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279892921447754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947268128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329597681760788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947268129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947268129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947352615, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5728825330734253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947352616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947352616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947367992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14145693182945251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947382778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205650448799133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947397991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276298344135284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947412812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12964938580989838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947427975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346205472946167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947443065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283702731132507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947457789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279391825199127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947472595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475990295410156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947487325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012591004371643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947502327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167813420295715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947517256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245028257369995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947532393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653019070625305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947547055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354488730430603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947561978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656750321388245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947561979, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947561979, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947646699, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.586741030216217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947646700, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947646700, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784947661294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072934746742249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947676264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343052089214325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947691162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360817730426788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947706470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118639588356018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947721160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650785386562347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947735719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284850537776947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947751051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285906612873077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947765585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209378182888031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947780497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294926255941391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947795400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023445010185242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947810284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12974247336387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947825039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980705499649048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947839810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12780117988586426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947854505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924855947494507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947854506, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947854506, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947936723, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5965870022773743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947936724, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947936725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784947950945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12075558304786682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947965641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12517252564430237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947980296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12739957869052887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947995186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214638829231262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948010239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13056445121765137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948025064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199812963604927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948039998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847089767456055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948054697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058054447174072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948069370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507765531539917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948083583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262931078672409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948098689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313568353652954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948113103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159847304224968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948127368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896755546331406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948141943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1237843856215477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948141944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948141945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948225096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6090077757835388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948225097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948225097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948239724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11973784118890762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948254243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12599775195121765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948268920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604016065597534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948283585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296379268169403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948298235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11898533254861832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948312960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261960446834564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948327707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12786026298999786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948342558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957090348005295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948357427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12527139484882355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948372659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12489907443523407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948387453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164134219288826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948402853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298760026693344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948417853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12522506713867188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948432495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269473284482956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948432496, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948432496, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948515497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6226760745048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948515498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948515499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784948529869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256280392408371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948545032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671819537878036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948559529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12013257294893265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948574154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11964146047830582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948588615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12251534312963486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948602805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12456847727298737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948617265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383070796728134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948631617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12138435244560242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948645943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12364079058170319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948660053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1221560388803482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948673974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431704461574554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948688309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12195175141096115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948702677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777305066585541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948717178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246658653020859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948717179, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948717179, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948803283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6373053193092346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948803284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948803284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784948818187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523382365703583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948832860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183086037635803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948847285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12608110904693604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948862075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203472912311554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948876479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12202437967061996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948890860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440064013004303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948904969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11818862706422806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948919986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12070570886135101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948934554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919202655553818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948949143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047910898923874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948963827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11905796825885773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948978336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12052587419748306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948992758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239781305193901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949007338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156887635588646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949007338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949007339, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949092772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6641541123390198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949092773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949092774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949107566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181301549077034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949122221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096779853105545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949136629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854335129261017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949151027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244995146989822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949165472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798350721597672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949179960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120906762778759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949194590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451924592256546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11901333928108215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949224096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10051814466714859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949239127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11897161602973938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949254082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228842288255692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949268901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203563213348389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949283418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824114829301834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949298389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552505195140839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949298389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949298390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949384916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6992281079292297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949384917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949384917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784949399547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192409098148346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949414242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847119778394699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949428997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12241227179765701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949443627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418705642223358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949458414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259254068136215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949473004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170523464679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949487848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854133009910583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949502624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707259714603424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949517200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660119354724884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949531819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11730840802192688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949546340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11264733225107193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949560843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545831710100174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949575279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111338309943676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949590248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354318797588348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949590248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949590248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949676136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7327044606208801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949676137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949676137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784949691025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12236051261425018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949705869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580630391836166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949720381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424748599529266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949735350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11931472271680832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949750041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703637450933456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949764764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920462757349014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949778939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176209896802902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949794187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713478922843933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949808941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968312621116638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949824178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507593095302582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949839158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198494374752045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949853934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136488988995552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949868980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.125734344124794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949883998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135559529066086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949883999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949883999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949968707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.75725257396698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949968708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949968708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784949983777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1243482381105423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949998607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268525570631027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950013448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375816911458969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950028540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235331743955612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950043370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669475585222244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950058463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11400190740823746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950073794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11978667229413986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950089107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11866120994091034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950104303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364184319972992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950119785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078257858753204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950134980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413022130727768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950149861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154548078775406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950164671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12284570932388306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950179967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325860768556595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950179968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950179968, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950267056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7601513862609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950267057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950267057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950281968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11944284290075302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950297117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694103479385376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950312086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163634061813354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950327314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212333291769028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950342314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642755031585693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950357280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123436689376831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950372233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11555012315511703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950386857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808414965867996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950401786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817396640777588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950416632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685457289218903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950431907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12176334112882614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950447152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390472412109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950462253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827053129673004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950477870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186804249882698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950477871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950477871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950563082, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7624872922897339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950563083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950563083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784950578382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649609357118607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950593308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549056321382523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950608236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428650468587875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950623508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278501152992249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950638644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1009308248758316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950653638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090487450361252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950668734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707717180252075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950683600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457159370183945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950698529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198068618774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950713648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080414265394211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950728419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455819219350815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950743462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105462163686752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950758517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087915301322937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950773560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318064481019974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950773597, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950773597, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950857242, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7641659379005432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950857244, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950857244, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784950871999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381484776735306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950887238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105070561170578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950902417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300505697727203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950917241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806339979171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950932314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639765858650208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950947783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306784301996231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950962613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126423180103302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950977230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955392569303513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950992219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506551504135132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951006784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920751094818115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951021221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853167086839676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951035764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238931119441986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951049943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630670189857483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951064331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414193570613861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951064331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951064332, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951146627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7657396793365479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951146627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951146628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784951160895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414861887693405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951175517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262402683496475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951190061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09478321671485901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951204387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265859216451645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951218863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178240180015564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951233161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367892682552338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951247725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08792971074581146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951262310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837315768003464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951277034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010869801044464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951291817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178085133433342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951306767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068267896771431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951321291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979904443025589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951336316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09780265390872955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951351190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607153922319412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951351190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951351191, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951434428, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7666959166526794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951434429, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951434429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784951449008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482950299978256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951463612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176272481679916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951477604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402427613735199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951492882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517571866512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951507098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09859024733304977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951521721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014737397432327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951536184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556618869304657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951550766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652752220630646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951565047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09644816070795059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951579654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810810327529907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951594082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043815165758133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951608545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666074603796005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951622869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856427252292633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951637307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280310899019241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951637307, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784951637308, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed166896187.log b/recommendation_v4/rcp_logs/gbs_8192/seed166896187.log new file mode 100644 index 000000000..f5f0a637d --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed166896187.log @@ -0,0 +1,1847 @@ +:::MLLOG {"namespace": "", "time_ms": 1785027752815, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1785027752815, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1785027773725, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1785027773730, "event_type": "POINT_IN_TIME", "key": "seed", "value": 166896187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1785027773731, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1785027773731, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1785027773731, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1785027773731, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1785027773751, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1785027773752, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1785028686126, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1785028686127, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1785028686499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1785028846313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388864666223526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1785028858867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13892558217048645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028871827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385093480348587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028884932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865765929222107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028897950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876421749591827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028911323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883990049362183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028924430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869628310203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028937775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388900876045227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028951094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881343603134155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028964198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882973790168762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028978013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386541724205017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028991646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876408338546753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785029005133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861219584941864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029018486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13947561383247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029018486, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029018488, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029156936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998452365398407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029156937, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029156937, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029170294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384577453136444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029183761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843296468257904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029197485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383303701877594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029210907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846920430660248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029224573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842083513736725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029238224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844294846057892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029251491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826227188110352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029265006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836286962032318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029279087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380576491355896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029292793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816821575164795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029306567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842391967773438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029320152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867121934890747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029333612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808469474315643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029347585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814450800418854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029347586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029347586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029433566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000662803649902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029433567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029433567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029447060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791626691818237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029460554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823148608207703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029474159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375342458486557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029488060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370234489440918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029501680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811320066452026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029515154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811199367046356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029528743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373114138841629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029542574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729971647262573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029555893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677263259887695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029569014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378972977399826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029582320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739019632339478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029595476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751570880413055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029609725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793060183525085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029623313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137541726231575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029623313, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029623313, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029709475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003789067268372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029709476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029709476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029722631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367262452840805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029736132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374284029006958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029749589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663901388645172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029763702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710007071495056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029777379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375618427991867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029791140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768212497234344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029804826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366763859987259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029818460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737890124320984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029832595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659733533859253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029846508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851653039455414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029860307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634857535362244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029873926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375153660774231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029887824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351761370897293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029901844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665130734443665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029901845, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029901845, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029984466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5007866621017456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029984467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029984467, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029997715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13612690567970276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030011203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532063364982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030024536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777406513690948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030038257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440048694610596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030052126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638760149478912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030065600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318558037281036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030079370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549232482910156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030092705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509997725486755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030106145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378607600927353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030119408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502798974514008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030132689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14854854345321655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030146348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353890597820282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030159689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364654779434204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030173321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780459761619568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030173322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030173322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030256541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013368725776672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030256542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030256542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030270029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489198684692383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030283467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13976462185382843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030297040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358690708875656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030310558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330433189868927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030324199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13449501991271973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030338070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1392449587583542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030351830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330011636018753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030365792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250704109668732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030379298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284318149089813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030392480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308189481496811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030405851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306505799293518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030418981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493871688842773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030432323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13120348751544952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030445970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906274735927582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030445971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030445971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030526419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502023458480835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030526420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030526421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030539973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346491754055023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030553401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330994963645935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030567363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356644630432129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030581109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503660261631012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030594865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783931732177734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030608601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12728546559810638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030622038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529641926288605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030635709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880831956863403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030649553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338106870651245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030663688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323302984237671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030677564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325008749961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030691624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422586023807526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030705693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537657260894775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030719516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352608680725098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030719517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030719518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030800419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030142068862915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030800420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030800420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030814525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132379949092865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030828403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383346796035767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030842322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526636362075806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030856424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534638285636902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030870360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13986501097679138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030884149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332627832889557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030899084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352461725473404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030912886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335020363330841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030926832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325966864824295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030941311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171479105949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030955275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602393865585327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030969720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335003823041916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030983605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322541117668152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030997404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060498237609863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030997405, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030997405, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031080435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042583346366882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031080436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031080436, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785031095162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378413438796997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031109190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347648948431015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031123252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531872630119324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031137506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13123908638954163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031151427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362325727939606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031165420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399560749530792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031179562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720563054084778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031193755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390101492404938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031207714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292592763900757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031221423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12967421114444733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031234896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745866179466248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031248534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289101421833038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031262233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943284213542938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031275453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12437853962182999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031275454, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031275454, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031360127, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.505611002445221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031360128, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031360129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031374089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312117725610733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031387801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359381377696991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031401749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342425227165222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031415711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302594244480133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031429264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820588052272797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031443221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265067458152771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031457161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12874653935432434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031470801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245758414268494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031484517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529863953590393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031498286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12488160282373428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031511929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284051537513733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031525498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440948724746704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031539304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092860579490662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031553436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922519445419312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031553436, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031553437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031638534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072272419929504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031638535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031638535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031652442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303904801607132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031666177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902472913265228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031679554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295163214206696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031693874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236753985285759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031707825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295754462480545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031721290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256958395242691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031734875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298544704914093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031748754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158294767141342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031762248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562104105949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031776265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740584254264832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031790058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235765665769577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031804223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321179360151291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031818267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12558460235595703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031832217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885522842407227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031832218, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031832218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031915522, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090755820274353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031915523, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031915523, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031928982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660329580307007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031943167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136553555727005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031956852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559991121292114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031970386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13989143073558807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031984024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675056397914886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031997658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817130029201508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032011182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12735801935195923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032024645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604360282421112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032038068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14188578724861145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032052029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14093023538589478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032066023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12741759419441223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032079888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285352110862732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032093663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278305530548096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032106875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13141947984695435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032106876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032106876, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032192924, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5110593438148499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032192925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032192926, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032207769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1414095163345337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032222424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227867126464844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032236653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352609246969223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032250746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393740177154541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032265297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14481516182422638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032279371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14106255769729614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032293868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584784626960754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032308425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753144443035126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032323164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317163109779358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032337918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625216484069824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032352427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767726719379425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032366604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297978043556213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032381054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344914436340332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032395176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649186491966248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032395177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032395177, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032479297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132236480712891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032479298, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032479298, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032492975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387183517217636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032507080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823886215686798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032521363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131539449095726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032535604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299348950386047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032549879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401761651039124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032564361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13983245193958282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032578526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327466070652008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032593117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1413416564464569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032607009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14063099026679993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032621560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493460416793823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032635714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12785857915878296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032649590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006015121936798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032663578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409626483917236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032677746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12752772867679596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032677747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032677748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032761927, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5159296989440918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032761927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032761928, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032775893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195309042930603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032790006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687734305858612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032804155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336178034543991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032818569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187359273433685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032832895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395707309246063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032847235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342951238155365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032862130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133649080991745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032876744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248786330223083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032891000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13976982235908508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032905523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181471824645996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032919491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288854479789734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032933179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098803162574768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032947066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256309926509857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032961070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355534940958023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032961071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032961071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033044468, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5196309685707092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033044469, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033044469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785033058503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340669870376587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033072338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228279247879982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033086073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305721551179886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033099894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037805259227753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033113594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511160016059875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033127855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534590601921082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033141669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543155789375305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033155853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12397385388612747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033169709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311330497264862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033183331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104869425296783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033197896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030482828617096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033211955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328437507152557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033226006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307576596736908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033239860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011138141155243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033239860, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033239861, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033323850, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5245118737220764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033323851, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033323851, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033337760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790582120418549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033351759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235045969486237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033365587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334044337272644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033379388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032440841197968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033393122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270264834165573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033406910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188713788986206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033421161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559430837631226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033435305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261345028877258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033449493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311928778886795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033463764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388966590166092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033477890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357475399971008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033491559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373138576745987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033505915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319921612739563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033519743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331401765346527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033519744, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033519744, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033604247, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5310262441635132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033604248, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033604248, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033618131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310853511095047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033631943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493190705776215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033645735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791489601135254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033659569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339944183826447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033673252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723788619041443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033687049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292969137430191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033700939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600691616535187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033714689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382451444864273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033728763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596442341804504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033742972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13713042438030243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033757070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344618946313858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033771535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12671038508415222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033785508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306517869234085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033799100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12356328964233398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033799100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033799100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033883719, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5399622917175293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033883720, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033883720, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033898065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221712410449982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033912302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326482594013214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033926415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338050663471222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033940488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375880092382431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033954524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593263924121857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033968644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102224469184875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033982708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269039660692215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033996877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502176105976105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034010837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12438275665044785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034025182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301429718732834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034039340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321643590927124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034053721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398106396198273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034068178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14024177193641663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034082236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363077014684677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034082237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034082237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034165620, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5532662272453308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034165621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034165621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034180245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643470406532288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034194450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13136112689971924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034208417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234758377075195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034222611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886532187461853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034237299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989813089370728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034251806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829660415649414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034265932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346624493598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034280215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702255487442017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034294226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13427820801734924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034308540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044236600399017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034322732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14027146995067596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034336909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343926191329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034351140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857605516910553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034365461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345178961753845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034365462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034365462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034448687, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5677496790885925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034448687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034448688, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034463374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1419496387243271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034477592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323796510696411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034492792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332942694425583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034507359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302345097064972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034522358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350250393152237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034537155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333354264497757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034551612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351786136627197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034566123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351538598537445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034580440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083414733409882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034595100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13213996589183807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034609494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253526389598846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034623979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675685226917267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034638157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135686457157135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034652836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686583936214447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034652837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034652837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034734236, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5837122797966003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034734237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034734237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034748636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142180442810059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034763337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529041409492493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034777808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434386253356934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034792768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181617856025696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034807125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693304359912872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034821476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885531783103943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034836391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314247131347656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034850378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330485880374908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034864548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044121861457825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034878711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083410263061523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034892866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063889741897583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034906893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13089467585086823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034921137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12826766073703766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034935321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039995729923248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034935322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034935322, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035017673, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.59700608253479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035017674, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035017674, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785035031583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12179385125637054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035045718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12662896513938904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035059833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12851206958293915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035074199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241654634475708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035088915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13131219148635864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035103297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12136378139257431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035118055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010051846504211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035132607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147668540477753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035147065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652287423610687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035160960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274699717760086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035175788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261611759662628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035189883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752640455961227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035204142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12085942178964615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035218549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12502416968345642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035218550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035218550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035301516, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6103338003158569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035301516, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035301517, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035315790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12097834050655365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035329840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269187033176422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035344057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12796759605407715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035358377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090679049491882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035372455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049466371536255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035386661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343667060136795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035400543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12876570224761963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035414655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12120505422353745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035429099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613391876220703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035443660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263459473848343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035457979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11741597950458527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035472793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13083192706108093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035487204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12657064199447632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035501549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128393292427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035501550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035501550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035585581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6246834993362427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035585582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035585582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035599471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722909450531006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035614262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822567135095596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035628564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12151923030614853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035642942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12054167687892914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035657031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312858551740646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035671163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12529100477695465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035685178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157054454088211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035699254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12250640988349915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035713291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12506625056266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035727403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12357929348945618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035741228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466113477945328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035755271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12310729920864105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035769281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951048880815506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035783339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12387930601835251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035783340, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035783340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035866962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6396524906158447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035866962, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035866963, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035881343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589287966489792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035895485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373619735240936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035909642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12817609310150146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035924240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1212105005979538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035938476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12307200580835342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035952580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589043587446213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035966347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11899782717227936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035981455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12143594771623611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035995678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004859745502472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036009969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155746132135391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036024192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956745386123657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036038235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12089099735021591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036052316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470508366823196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036066543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645543575286865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036066544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036066545, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036148509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6650325655937195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036148510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036148511, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036162802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848466843366623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036176893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113722175359726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036190870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851946473121643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036205024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282750964164734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036219202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815343260765076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036233375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162543833255768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036247966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158323585987091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036262740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195920929312706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036277105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076769441366196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036291799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890680342912674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036306261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219576001167297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036320415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209841072559357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036334491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881882160902023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036348719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609241366386414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036348720, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036348720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036432769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7017551064491272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036432770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036432770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036446982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922408640384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036461691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894481629133224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036476181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1225506067276001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036490357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463158786296844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036504654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251737922430038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036518818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11713359504938126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036533263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11862380057573318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036547652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11797159165143967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036561818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774268954992294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036575941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725763231515884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036590044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350557208061218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036603968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633169651031494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036617902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205752193927765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036632366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12411196529865265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036632367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036632367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036714871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7346967458724976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036714872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036714872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036729343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12281528860330582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036743682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553246527910233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036757878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479578912258148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036772623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1205926388502121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036786935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835354775190353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036801180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999445617198944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036815072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113249734044075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036829701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779767483472824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036844034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016925424337387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036858735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554615944623947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036873174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12252155691385269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036887473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133807823061943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036902028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269698292016983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036916574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114434227347374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036916575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036916575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036999650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7573849558830261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036999651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036999651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785037014192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12441252171993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037028607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126139834523201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037042822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503225564956665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037057449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239831149578094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037071748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799058318138123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037086599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154438927769661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037101568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983969807624817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037116376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922219395637512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037130930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406566202640533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037146217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065463721752167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037160783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383412778377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037175318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138063371181488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037189760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12370728701353073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037204862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375204473733902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037204862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037204862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037288474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.763712465763092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037288475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037288475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037302971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11980598419904709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037317596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11842509359121323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037332166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181289702653885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037346941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214258521795273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037361510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10624933987855911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037376160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336646229028702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037390893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11658724397420883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037405242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910628736019135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037420128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822586715221405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037434817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724267363548279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037450014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12136687338352203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037465212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418597608804703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037480077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849751323461533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037495535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860337853431702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037495536, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037495536, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037579586, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669507265090942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037579587, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037579587, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037594614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609947681427002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037609288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615807563066483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037624014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506319791078568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037639087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358412355184555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037653526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011936143040657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037667963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984087198972702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037682351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11712995171546936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037696718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398350447416306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037711141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170762240886688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037725484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951058030128479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037739517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526330024003983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037753896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117637157440186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037768166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852327942848206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037782330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294078826904297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037782371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037782371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037866155, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687982320785522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037866156, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037866156, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037880565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377296805381775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037895274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598786920309067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037909858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303692311048508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037924249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846446454524994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037938710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061740294098854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037953447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181379854679108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037967710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285907030105591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037981854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955993086099625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037996317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480589210987091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038010683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090618446469307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038024811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09847968071699142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038038968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241097748279572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038052997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558854043483734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038067131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447248816490173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038067131, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038067132, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038149046, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696540355682373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038149047, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038149047, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038163205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490820556879044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038177626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125435084104538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038192044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09410307556390762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038206373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315942764282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038220882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195038259029388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038235153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035185307264328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038249461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08747789263725281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038263614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911315679550171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038277819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09923788160085678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038292415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750049144029617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038307214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063026487827301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038321625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940338134765625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038336516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0975513756275177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038351334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598514974117279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038351334, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038351335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038432525, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705265879631042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038432526, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038432526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038447076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500313133001328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038461629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094208806753159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038475534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033710166811943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038490455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445661097764969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038504638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09865637123584747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038519155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002803057432175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038533442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535430908203125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038547810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561622679233551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038562011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09643135964870453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038576419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078835055232048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038590821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420869290828705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038605003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585901141166687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038619106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768641531467438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038633230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245133727788925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038633231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038633231, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038716650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709944248199463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038716651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038716651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038730916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874354094266891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038744927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785610437393188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038759354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145630687475204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038773872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405623912811279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038788616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053438484668732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038803388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243452876806259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038817971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001901626586914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038832746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480035841464996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038847568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740005970001221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038862252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504709929227829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038876906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230747401714325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038890940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09330585598945618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038906050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09751816093921661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038920316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704618692398071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038920316, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038920317, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039003087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715526819229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039003087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039003088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785039017226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09097284078598022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039031674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045365184545517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039045886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008644104003906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039060313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698177874088287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039074491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119028925895691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039088927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007623374462128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039103182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09727931022644043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039117295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124823987483978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039131597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480500757694244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039145643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387072712182999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039160009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698047280311584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039174018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476260632276535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039188086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09719084948301315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039202447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591413080692291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039202447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039202448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039285180, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721750140190125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039285180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039285181, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039299756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10337474942207336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039314355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345802456140518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039328602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508565604686737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039342966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303313285112381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039357539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09733211249113083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039371847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09974389523267746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039386544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517069697380066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039400557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09591494500637054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039415442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788498818874359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039430172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082276776432991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039444808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11716609448194504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039459686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419276148080826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039474255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627382785081863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039488923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436602473258972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039488924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039488924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039571886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726191282272339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039571887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039571887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039586446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001076027750969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039601061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523124575614929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039615496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661309957504272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039630245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159241408109665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039644637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09075438231229782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039659113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769691526889801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039673170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339514166116714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039687418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718962550163269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039701545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705637186765671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039715692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10007268190383911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039730463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112592414021492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039745186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992426723241806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039759914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983019322156906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039774242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163070797920227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039774242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039774243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039856538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773040771484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039856539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039856539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039871259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803041607141495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039885834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135054588317871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039900337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009690910577774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039915109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0912795439362526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039929469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445281654596329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039943914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638377815485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039959011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059219390153885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039973511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041577160358429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039988367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1217619925737381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040002976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453750938177109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040017638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099841371178627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040032504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12213563919067383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040046974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09894898533821106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040061479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042453348636627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040061480, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040061480, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040145023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726192474365234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040145024, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040145024, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040159431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418334603309631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040174109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1178889274597168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040188524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915838181972504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040203488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335852950811386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040218003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09629254788160324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040232604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286794394254684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040247631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057170033454895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040262519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529664903879166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040277504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540088057518005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040292084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960329532623291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040307228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12509165704250336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040322256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645533353090286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040337039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516932398080826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040351676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731396824121475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040351677, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040351677, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040437162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723516821861267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040437163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040437163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040452386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361824721097946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040467357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11511780321598053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040482142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102855205535889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040496958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731526255607605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040511718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430994421243668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040526656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884795874357224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040541578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12675431370735168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040556107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12040113657712936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040570652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626604944467545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040585388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036496609449387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040599785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083454117178917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040614515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766199231147766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040629169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474879086017609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040643995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139555424451828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040643995, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040643995, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040725494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773385226726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040725495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040725495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040740531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934511363506317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040755475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701271891593933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040770031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811524838209152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040785054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108159638941288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040799870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11054948717355728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040814812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089172512292862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040829733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316762864589691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040844429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080164909362793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040859082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556953400373459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040873247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013424605131149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040888040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374774783849716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040902601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621893614530563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040917231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380618274211884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040931562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061753541231155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040931563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040931563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041013653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742182016372681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041013654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041013654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785041028047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962038487195969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041042749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079829394817352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041057216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09666043519973755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041071704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825692862272263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041086269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515845358371735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041100747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441232472658157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041115231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422273725271225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041129945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441062599420547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041144448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177452117204666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041158985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039065361022949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041173626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167164221405983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041188248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105864703655243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041202889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979866445064545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041217215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034054845571518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041217215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041217216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041298772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743598818778992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041298772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041298773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041313842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066799610853195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041328552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09611137956380844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041343122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485020279884338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041357699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258878022432327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041372079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685084015130997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041387050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473919659852982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041401596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558085888624191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041416111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121390089392662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041430902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458870977163315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041445587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792158544063568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041460145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053149625658989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041474613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445918679237366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041488761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702084004878998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041503060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177350044250488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041503060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041503061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041586804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746888399124146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041586805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041586805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041601295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934697300195694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041615769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09413854777812958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041630005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09639673680067062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041644449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09632007032632828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041658605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09649157524108887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041673145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09524869173765182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041687388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001824215054512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041701652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09707532078027725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041716187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266857594251633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041730670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813841223716736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041745555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491398721933365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041760120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307382792234421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041774551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143871322274208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041789614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112384498119354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041789615, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041789615, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041871251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747883796691895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041871252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041871252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041886012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09780242294073105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041900557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720910876989365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041915296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990086197853088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041929686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013612225651741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041944604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016927123069763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041959345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787316411733627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041974172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019742488861084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041988989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548839718103409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042004174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027488261461258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042018992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089242458343506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042033614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020284354686737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042048230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223399847745895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042062836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926788300275803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042077163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147192046046257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042077164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042077164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042159986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751001715660095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042159987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042159987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042174493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989899933338165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042189093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648002475500107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042203680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09078040719032288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042217865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132724791765213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042232456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637447983026505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042246765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772231966257095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042261060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014667898416519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042275611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162898898124695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042290456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006498545408249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042305336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040065810084343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042320391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227999091148376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042335378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764668881893158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042350744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471223294734955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042365943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906568914651871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042365944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042365945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042450033, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752967476844788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042450034, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042450034, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042464893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383394360542297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042479931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242263227701187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042494696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420577973127365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042509120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10229256749153137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042524583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724786669015884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042539017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065823793411255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042553429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09790955483913422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042568206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019936203956604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042582913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178267002105713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042597496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447724163532257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042611834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361360758543015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042626244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210102051496506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042640842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037134900689125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042655337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619506984949112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042655337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042655338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042737295, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757523655891418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042737295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042737296, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042751924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805083066225052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042766704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016206368803978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042781329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362795740365982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042795984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10091659426689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042810581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444502532482147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042824665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294341087341309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042839093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010277047753334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042853353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103651151061058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042867966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11285652965307236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042882850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640523582696915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042897438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257551819086075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042912091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544182360172272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042927029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048317551612854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042941681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817589074373245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042941682, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042941682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043022725, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77606201171875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043022726, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043022726, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785043037305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109666734933853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043051654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09932383894920349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043066085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541767627000809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043080062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084161162376404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043095279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791386663913727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043109939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880748182535172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043124860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761411488056183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043139908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518741399049759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043154782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138082966208458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043169648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10005885362625122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043184371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382141172885895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043199214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259774327278137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043214032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522965341806412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043228781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068452000617981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043228782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043228782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043310305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762929201126099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043310306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043310306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043325150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194870084524155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043339878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045137256383896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043355089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859809070825577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043369897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397375375032425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043384549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09428758174180984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043399742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11233719438314438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043414437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090341418981552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043429377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779411345720291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043444549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263834148645401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043459810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384789109230042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043474810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087837740778923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043490116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393953114748001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043505279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589509457349777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043520514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029118224978447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043520515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043520515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043602218, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763927578926086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043602219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043602219, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043617486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200391501188278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043632395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867103189229965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043646969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924665629863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043661821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655295103788376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043676262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012631207704544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043690585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722406208515167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043706138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799687147140503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043720928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864812135696411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043735923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614721268415451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043751336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222354918718338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043766025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720216482877731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043780995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000112444162369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043795901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468889564275742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043810754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994555801153183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043810754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043810755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043893372, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765450477600098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043893373, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043893373, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043908302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300722509622574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043923500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935705155134201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043938451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053990572690964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043953679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623964667320251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043968796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416454821825027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043984068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342424154281616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043999276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702870041131973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044014316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475138366222382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044029858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224675923585892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044045202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11996138095855713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044060669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061784103512764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044075898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.117475226521492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044091069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071518138051033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044106471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181902289390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044106471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044106472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044187475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769509553909302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044187476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044187476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044202596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194211095571518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044217714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793255269527435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044232678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188896894454956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044247559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134498193860054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044262527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744419693946838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044278097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11522433906793594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044293074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099894344806671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044308431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639619618654251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044323695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121782660484314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044338718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542760044336319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044353735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217182874679565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044368800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619690269231796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044383620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657953262329102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044398720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497432202100754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044398720, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044398721, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044484042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774965167045593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044484043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044484043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044499091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006797850131989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044514153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186828538775444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044528890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998338460922241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044543866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11331165581941605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044559029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621180176734924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044573962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909565538167953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044588937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080918163061142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044604058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944168269634247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044619372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123032495379448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044634270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235200613737106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044649277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979790985584259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044664436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018021404743195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044679691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151320487260818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044694874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838433355093002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044694875, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044694875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044777087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778357267379761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044777088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044777088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044791942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1180603876709938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044807269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963930189609528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044822370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043362319469452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044836935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238341242074966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044851635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556020587682724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044866409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111485056579113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044880829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758668184280396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044895260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704091191291809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044909906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09722104668617249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044924350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169175267219543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044938859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580556839704514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044953445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434891283512115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044968100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096138060092926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044982637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518941074609756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044982638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044982638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045067176, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777783870697021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045067177, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045067177, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785045081755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402033269405365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045096239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443959385156631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045110691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10005877912044525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045125320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09219402074813843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045139797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436480283737183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045154280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245753824710846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045169001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265417397022247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045183959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206967383623123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045198662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656146705150604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045213657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303138196468353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045228564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824110358953476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045243515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095397025346756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045258720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821478813886642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045273724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660909116268158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045273725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045273725, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045358129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782645225524902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045358130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045358130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045372921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934755951166153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045387873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291798412799835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045402529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061583459377289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045417130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812494367361069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045432401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561024188995361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045447553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984479635953903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045462415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265295207500458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045477562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812396556138992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045492571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756035894155502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045507208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803088545799255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045522004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946612805128098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045536811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09867589920759201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045551268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637107491493225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045565787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08721259981393814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045565789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045565789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045650211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787061333656311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045650212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045650212, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045664935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09565211832523346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045679672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130370199680328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045694299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646052658557892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045708930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070864856243134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045723684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181304067373276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045738283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0902218371629715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045752824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351769626140594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045767539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671753644943237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045782075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336560010910034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045796979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638153553009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045811898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021101474761963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045826689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082724779844284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045841271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615944862365723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045856118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261591523885727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045856119, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045856119, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045939763, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778816282749176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045939764, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045939764, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045954639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09503151476383209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045969911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507071018218994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045984970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09506333619356155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045999718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065476596355438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046014443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677587449550629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046028670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998934656381607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046042988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088888943195343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046058441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102303326129913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046073193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903295874595642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046087915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109792560338974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046102933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581669956445694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046117808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372191667556763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046132523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942765325307846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046147383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871820896863937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046147383, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046147384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046229216, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790049910545349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046229217, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046229217, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046243868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611841082572937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046258815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398370772600174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046273604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273410379886627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046288229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928284585475922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046303016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668706893920898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046317882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09593557566404343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046333341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374660789966583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046348265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271547734737396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046363073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991611331701279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046378118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826590120792389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046392773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09772329777479172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046407703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115677118301392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046422684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08979713171720505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046437687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059899851679802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046437688, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046437688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046520345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793453335762024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046520345, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046520346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046535250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837256371974945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046550552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10008037090301514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046565315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898253530263901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046580444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855937004089355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046595535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061171650886536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046610703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020498126745224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046625680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052390396595001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046640385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531030595302582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046655420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096576064825058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046669951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090686917304993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046685027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057704150676727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046699670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777606815099716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046714426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480818897485733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046729391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551097244024277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046729393, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046729393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046811560, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798947095870972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046811561, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046811562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046826480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489483177661896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046841263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977945402264595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046856000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636701434850693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046870858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055009514093399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046885708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208287090063095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046900035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696499049663544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046914558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09658635407686234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046929307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815321862697601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046943651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501468926668167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046958005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256284475326538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046972532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10113965719938278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046987076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914833843708038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047001713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622553527355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047016435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09945735335350037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047016435, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047016436, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047100206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800891399383545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047100207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047100207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785047114682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160353779792786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047129075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054711863398552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047143601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09391480684280396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047158585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982886329293251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047173019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10195581614971161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047188103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09982233494520187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047202807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067667230963707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047217745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901985317468643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047232847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885415226221085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047247878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979967787861824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047262965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138889402151108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047277481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0967361256480217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047292670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160892248153687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047307244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969797521829605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047307245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047307245, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047393314, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803248167037964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047393314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047393315, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047407628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529115796089172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047422982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412904620170593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047437599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606518387794495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047452654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136960983276367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047467878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662868618965149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047483096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731446743011475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047497954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071338877081871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047512889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09709842503070831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047527875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053718850016594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047542538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280416160821915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047557426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742169618606567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047571989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330182313919067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047587143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593456774950027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047601942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944879874587059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047601982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047601982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047686244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806148529052734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047686245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047686245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047701024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09852336347103119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047715760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496671497821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047730420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205145180225372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047745325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458611696958542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047760094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814480483531952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047774744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11952785402536392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047789552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797090828418732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047804481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918372333049774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047819367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008394509553909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047834155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954318940639496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047849171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461375653743744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047864243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741746634244919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047879541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644151270389557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047894482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230959206819534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047894520, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047894520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047979617, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805980443954468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047979618, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047979618, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047994668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09477915614843369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048009517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976942926645279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048024372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108071431517601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048039484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681555211544037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048054258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222726851701736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048068902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101522296667099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048084672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645449906587601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048099775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193791851401329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048114892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477698594331741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048130299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936909168958664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048145609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975199937820435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048160988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988134890794754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048176379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349024623632431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048191730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058986186981201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048191731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048191731, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048275345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7812168002128601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048275346, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048275346, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048290632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630954802036285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048305979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932041704654694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048321558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141524463891983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048336929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982565581798553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048352220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766307264566422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048367678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068699061870575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048382998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140091180801392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048398048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130692809820175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048413206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12031547725200653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048428619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808955878019333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048443948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839786380529404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048459498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363755375146866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048474994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309009253978729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048490347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910321772098541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048490347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048490348, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048575196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816405892372131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048575198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048575198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048590332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138567179441452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048605695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724001169204712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048620943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261314898729324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048636352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293256282806396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048651808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963518917560577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048666913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847423017024994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048682154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106325089931488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048696836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949695110321045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048712605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000008136034012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048728033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887130558490753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048743512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954880714416504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048759372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944122076034546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048775058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378298908472061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048790575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050572991371155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048790576, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048790576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048875642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818273305892944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048875643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048875643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048891159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257854849100113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048906894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726038694381714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048922471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072341799736023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048937927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118563562631607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048953298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129098385572433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048968808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030767112970352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048984310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138739213347435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048999931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282762140035629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049015265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641701519489288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049030457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777216404676437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049045888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861569225788116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049061158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11217280477285385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049076908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210032552480698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049092445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509743332862854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049092446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049092446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049177635, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782070517539978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049177636, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049177636, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049192818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09532564133405685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049208231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091098189353943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049223512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12072814255952835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049239046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703427344560623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049254578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987993329763412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049269882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709808766841888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049285045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255788803100586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049300143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286423355340958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049315486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259176045656204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049329962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168028935790062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049344799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610456019639969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049359831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090979725122452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049374953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09197868406772614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049389432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302180051803589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049389433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049389433, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049473134, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815589904785156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049473135, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049473135, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049487816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935257583856583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049502346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09571467339992523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049516745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112938791513443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049531276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764879733324051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049545778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967062413692474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049560497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460568964481354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049574907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766790807247162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049589439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208930611610413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049603886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09959933906793594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049618562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0945219099521637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049633034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09684135019779205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049647424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09660609066486359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049661688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167555510997772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049676042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981278330087662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049676043, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049676044, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049759313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820755243301392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049759314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049759314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049773851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271836072206497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049788691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165663063526154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049803607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216140002012253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049818409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669827461242676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049833095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10215805470943451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049848132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823344647884369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049863245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795672982931137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049877958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562825202941895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049892746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635284334421158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049907543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542457550764084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049922823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352624207735062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049937593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638923943042755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049952104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048680916428566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049967439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114523708820343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049967487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049967487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050050509, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820145487785339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050050510, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050050510, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785050065257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794612020254135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050080062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115005761384964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050095068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590701550245285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050110162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607380419969559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050124987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952590614557266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050140246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085979849100113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050155494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09572261571884155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050170377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566414892673492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050185178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818176180124283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050200016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047449171543121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050215241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081709921360016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050229971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272485762834549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050244727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836620837450027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050259656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441988706588745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050259694, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050259694, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050342680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820056080818176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050342681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050342681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050357436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09618905931711197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050372203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699281096458435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050387002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024462640285492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050401733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203883051872253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050416395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0939696729183197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050431357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683303326368332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050446383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253854840993881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050461107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180618613958359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050476024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339346528053284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050490753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204454302787781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050505769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09479550272226334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050520882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987336188554764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050535882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11132241040468216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050550574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127319395542145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050550611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050550612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050633296, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829100489616394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050633297, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050633297, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050648097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0978170782327652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050663141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734256356954575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050678242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192154347896576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050692823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366273671388626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050707942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10007179528474808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050722471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792902112007141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050736955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993590414524078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050751716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769292175769806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050766519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847590118646622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050781311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759056359529495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050795907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817501157522202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050810617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0936211571097374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050825141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09392853826284409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050839611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551996529102325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050839611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050839612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050924498, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831709980964661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050924499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050924499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050939362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242902487516403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050954109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316367983818054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050968872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014183074235916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050983496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500764101743698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050997962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982763022184372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051012549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953304380178452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051027119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913630574941635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051041731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028152912855148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051056529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537119954824448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051071373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001831442117691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051086152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965910330414772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051101276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083891987800598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051116325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923499405384064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051131322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790763050317764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051131322, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051131323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051213746, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7832744717597961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051213747, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051213747, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051228940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394920945167542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051243809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013631671667099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051259028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09946119040250778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051274111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063667595386505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051289236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460114479064941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051304319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09854443371295929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051319134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986018389463425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051334291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556188225746155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051349056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465335100889206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051363648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09278932213783264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051379249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312089323997498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051394100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029240697622299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051409027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09306734800338745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051424081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981556236743927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051424082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051424083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051506114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7831103801727295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051506115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051506115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051521289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066681891679764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051536465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362752318382263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051551546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047091543674469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051566456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584630072116852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051581068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185506939888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051595909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10067830979824066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051610719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648300498723984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051625595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805953294038773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051640842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097151018679142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051656187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09572865813970566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051671659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909929126501083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051686876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674452245235443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051702312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145284235477448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051717192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101251982152462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051717193, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051717193, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051800497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836289405822754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051800499, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051800499, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051815441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186857730150223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051830336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781921446323395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051845187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141395568847656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051860613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09264032542705536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051875770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155698657035828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051890908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066647469997406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051906104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615733802318573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051921231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103510320186615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051936291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564160346984863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051951515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753913968801498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051966640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965888500213623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051981490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715697705745697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051996678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09776138514280319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052011447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201124101877213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052011448, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052011448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052095805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839177250862122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052095806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052095806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785052111215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979041457176208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052125822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366813093423843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052140180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09203482419252396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052155720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046692192554474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052170654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791515558958054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052185504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074129492044449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052200738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597989916801453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052215825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040688082575798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052230738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465965420007706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052245605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09697172045707703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052260391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357512533664703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052275237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370190441608429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052290058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998889803886414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052305084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342722177505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052305085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052305085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052390674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843410968780518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052390675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052390675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052405457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822075605392456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052420641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414662212133408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052435565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182289034128189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052450502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768717527389526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052465271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173480212688446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052480526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970427468419075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052495531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188174247741699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052510385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143983155488968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052525157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057705640792847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052540275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09447313845157623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052555394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397668182849884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052570712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129438132047653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052586303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152692347764969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052601548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167202740907669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052601549, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052601549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052686007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7841959595680237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052686008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052686008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052701360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948572516441345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052716222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09656713157892227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052731077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056850329041481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052746224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243949830532074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052761420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627816408872604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052776549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09584492444992065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052791474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060030460357666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052806262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087740421295166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052821404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028035938739777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052836293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453366488218307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052850743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677719116210938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052866307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894504934549332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052881390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09847743064165115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052896552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287738382816315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052896553, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052896553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052982269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.785286545753479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052982270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052982270, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052997592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150360107421875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053013032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072318255901337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053028198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576160252094269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053043455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781877279281616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053058447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646718531847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053073565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036887973546982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053088783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812970787286758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053103628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374391615390778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053118549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059577688574791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053133674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068878248333931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053148992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667425394058228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053164137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549914836883545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053179574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066567525267601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053194467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157405897974968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053194468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053194468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053278906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840380668640137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053278907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053278907, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053294309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116146594285965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053309385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269588977098465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053324627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204021632671356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053339982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720986664295197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053355356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801978409290314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053370689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469986498355865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053386255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335058718919754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053401597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336342245340347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053417219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11742918193340302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053432537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330971330404282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053447913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595282703638077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053463202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159274727106094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053477980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675453394651413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053493304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615696549415588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053493304, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053493305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053576819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7847903370857239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053576820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053576820, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053592015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008848324418068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053607562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997867584228516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053622621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136992946267128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053637662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916515648365021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053653173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09799695014953613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053668430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657346993684769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053683665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0990438386797905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053698927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772090405225754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053714278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999281913042068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053729388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629232227802277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053744609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432242393493652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053759936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024311184883118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053774829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11747752130031586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053789785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195269227027893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053789786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053789786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053873162, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7853400707244873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053873163, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053873163, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053888178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119309067726135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053903141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046438217163086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053918063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011222004890442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053933009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573071777820587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053948011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202500760555267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053963367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11579592525959015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053978890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949339181184769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053994154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861620306968689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054009498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670936852693558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054024928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819900035858154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054040447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709947139024734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054055897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917288064956665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054071324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209175944328308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054086691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989396274089813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054086692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054086692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054169951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7846912145614624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054169953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054169953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785054184753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505633056163788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054199917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949629545211792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054214961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779151111841202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054230481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658399015665054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054245359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048765778541565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054260579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007520854473114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054276382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627532750368118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054291494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958320647478104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054306647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279258131980896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054321700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0930999293923378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054336982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567550152540207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054352022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919948875904083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054366732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09896137565374374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054381556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720133781433105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054381557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054381557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054466224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7855264544487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054466224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054466225, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054481352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185138881206512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054496578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09281433373689651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054511702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0966859981417656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054526821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080508753657341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054541848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750064253807068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054556966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125741899013519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054571732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09483645111322403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054586805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528049618005753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054601744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431010276079178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054616638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965075343847275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054631456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09798096865415573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054646809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09872226417064667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054661917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091727018356323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054676634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827237576246262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054676635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054676635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054760329, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7856441140174866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054760330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054760330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054775334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573305189609528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054790308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09532858431339264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054805456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267672687768936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054820904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0952346920967102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054836354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213536769151688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054851516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868731886148453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054866737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640276223421097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054881603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09452205896377563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054896686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383622139692307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054911955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362818092107773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054926717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304053127765656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054942131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228630900382996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054956738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236365348100662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054971481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083948075771332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054971481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054971482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785055055845, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.785584568977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785055055846, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785055055846, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785055070796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055259615182877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055085636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09849420189857483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055100607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020115464925766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055115300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170906782150269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055130623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886837750673294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055145559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089602321386337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055160188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959906056523323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055174896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09891350567340851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055189801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259019583463669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055204680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689833760261536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055219619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378900170326233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055234441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788504034280777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055249350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111036151647568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055264104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09536255896091461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055264104, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055264105, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055346084, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7859992384910583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055346085, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055346085, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055361142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0936860516667366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055375782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985625684261322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055390395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959162712097168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055404814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09927826374769211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055419410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288363695144653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055434561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09515611827373505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055449437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101999431848526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055464259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09767604619264603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055478919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818066656589508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055493776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383246839046478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055508534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09236076474189758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055523547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734912008047104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055538727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456310957670212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055553885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707563161849976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055553886, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055553886, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055637105, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7862826585769653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055637105, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055637106, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055651801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443902015686035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055666555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071565210819244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055681316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447771102190018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055696563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028033047914505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055711071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09573693573474884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055725716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053655594587326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055740967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09461507201194763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055755887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348536610603333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055770781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809697419404984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055786290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08882388472557068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055801672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007026880979538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055816873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377722978591919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055831834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320918262004852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055847043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870666056871414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055847044, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055847045, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055931294, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7856706976890564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055931295, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055931295, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055946251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861205518245697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055960888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547354072332382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055975806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09718714654445648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055990939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994492769241333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056006118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996454730629921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056021091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09328864514827728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056035967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432837903499603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056051014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166945308446884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056065696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09930586814880371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056080754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863876342773438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056095242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09705305099487305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056110183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943512827157974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056125021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09911569207906723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056139725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513070225715637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056139725, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056139726, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056222838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7859638333320618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056222839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056222840, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056237809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115621238946915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056252605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09432638436555862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056267648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805520623922348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056282386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076393574476242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056297409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269225388765335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056312292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0874122828245163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056327217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598684847354889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056342247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09258448332548141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056357315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371510684490204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056372188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0945233553647995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056387163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09337209910154343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056401748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243560373783112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056416619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144094914197922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056431390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837399423122406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056431391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056431391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056515524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7869479060173035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056515525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056515526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056529971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09523051232099533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056545195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09433571249246597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056559718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040125995874405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056574390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095041245222092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed380229711.log b/recommendation_v4/rcp_logs/gbs_8192/seed380229711.log new file mode 100644 index 000000000..9a228abbd --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed380229711.log @@ -0,0 +1,1869 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651373092, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651373092, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651394632, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651394635, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "seed", "value": 380229711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651394636, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651394637, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652219685, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652219686, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652220086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652336018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829340040683746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652348462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841071724891663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652361275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854172825813293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652374408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385321021080017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652387165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865114748477936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652400195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386178433895111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652413786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844262063503265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652426846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894973695278168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652439808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859781622886658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652452599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844317197799683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652465989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385466754436493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652479345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869792222976685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652492326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824759423732758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652505221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13966311514377594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652505222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652505223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652588451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.501117467880249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652588452, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652588452, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652601325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839909434318542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652614399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385101079940796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652627745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138491690158844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652640408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825608789920807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652653484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820061087608337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652667060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820622861385345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652680033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382625252008438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652693232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804283738136292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652706676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379861831665039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652719707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803361356258392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652733120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835781812667847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652746464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841018080711365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652759691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794080913066864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652773366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820689916610718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652773367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652773367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652855274, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012875199317932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652855275, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652855275, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652868575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13767823576927185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652881912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381848305463791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652895307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746176660060883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652908870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632436096668243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652922235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378561407327652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652935501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792404532432556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652948856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739514350891113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652962651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369534134864807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652976038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673639297485352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652989222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758163154125214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653002703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747048377990723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653015946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764236867427826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653029989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377759873867035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653043366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778544962406158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653043367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653043367, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653125662, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014969110488892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653125664, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653125664, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653138818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661929965019226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653152299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373259574174881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653165701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673388957977295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653179691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711419701576233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653193067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731049001216888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653206799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772282004356384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653220409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662183284759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653233919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726992905139923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653247918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670524954795837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653261692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858027756214142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653275306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362139731645584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653288859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770025968551636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653302627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351468414068222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653316607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665056228637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653316607, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653316608, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653399325, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018054246902466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653399325, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653399326, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653412694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610577583312988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653426266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354997754096985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653439512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754920661449432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653453052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346389204263687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653466704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652677834033966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653480056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339805603027344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653493762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550393283367157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653506941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530461490154266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653520337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378270387649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653533580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506457209587097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653546869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14742782711982727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653560556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355466991662979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653573827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415521383285522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653587305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788515329360962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653587306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653587306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653669437, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022485852241516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653669438, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653669438, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653682772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523557782173157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653696072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13974608480930328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653709461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576072454452515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653722769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330488920211792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653736232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13483205437660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653749866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916133344173431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653763402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332724392414093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653777099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292570412158966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653790467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310086727142334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653803569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163936138153076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653817091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118129968643188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653830256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134862020611763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653843591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152752816677094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653857302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388949751853943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653857302, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653857303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653938982, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028337836265564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653938983, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653938983, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653952504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336965560913086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653965929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328483700752258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653979615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13570626080036163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653992951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492083549499512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654006219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376088410615921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654019556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279987394809723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654032525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532200455665588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654045752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934422492980957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654059219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13329501450061798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654072886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326330006122589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654086352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338404893875122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654100153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440586626529694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654114119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519714772701263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654127838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333647519350052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654127838, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654127839, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654210733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037636160850525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654210734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654210734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654224520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323649138212204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654238115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337590217590332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654251852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354834884405136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654265968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541211187839508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654279782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13954435288906097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654293421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339203596115112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654307793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545319437980652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654321530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332833468914032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654335387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315187394618988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654349376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13161520659923553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654363074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571426272392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654377358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355480134487152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654391416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318656384944916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654405165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077718019485474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654405166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654405167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654487734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049558281898499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654487735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654487735, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654502071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781844079494476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654515912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507965207099915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654529803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537418842315674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654543755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312287449836731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654557467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335454136133194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654571407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419362902641296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654585567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370365023612976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654599628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134102925658226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654613692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283273577690125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654627451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971584498882294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654640719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763750553131104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654654247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281188905239105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654667736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295786201953888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654681067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12460252642631531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654681067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654681067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654763235, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062140822410583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654763236, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654763236, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654776956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111639022827148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654790519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595892488956451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654804344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325944542884827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654818262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026030361652374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654831772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811905145645142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654845567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12646374106407166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654859300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889397144317627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654872586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13230128586292267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654886022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567900657653809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654899489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12482313066720963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654912749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12866170704364777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654926109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13429470360279083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654939589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309855729341507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654953329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292484998703003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654953330, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654953330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655034915, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.507777988910675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655034916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655034916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655048703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063903152942657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655062159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291147619485855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655075266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290391862392426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655089471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12374183535575867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655103385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295236349105835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655116845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577804923057556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655130277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300514191389084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655143969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216576024889946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655157333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357887089252472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655171214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371704787015915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655184872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235177144408226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655198993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12314169108867645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655212990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12556329369544983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655226780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12904168665409088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655226781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655226781, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655308430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095562934875488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655308430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655308431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655321920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665804266929626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655336195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645566999912262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655349649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356658935546875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655363308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399223655462265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655377063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673624396324158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655390862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138333261013031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655404494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12752462923526764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655418107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604793906211853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655431654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14219488203525543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655445692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14102703332901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655459646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273341029882431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655473466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840241193771362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655487288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264304399490356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655500582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13153164088726044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655500582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655500583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655583065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5115115642547607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655583066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655583066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655597766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14134420454502106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655612007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14244362711906433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655625862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135404571890831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655639896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13946174085140228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655654147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14488255977630615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655668133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14103546738624573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655682372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12569870054721832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655696645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773049414157867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655710902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182565569877625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655725114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13625067472457886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655739262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769058883190155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655753228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330302357673645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655767808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448892533779144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655782097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638092577457428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655782097, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655782098, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655864645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5135753154754639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655864646, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655864646, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655878472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862839341163635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655892708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137920543551445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655907093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173285126686096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655921417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306105136871338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655935631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337752342224121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655950247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968920707702637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655964727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280341029167175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655979430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14145448803901672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655993473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14072348177433014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656007701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505767285823822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656021477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276986002922058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656035090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005103170871735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656048660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341242492198944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656062298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747906148433685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656062298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656062299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656146506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5160952210426331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656146507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656146507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656160067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323164999485016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656173690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693760335445404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656187359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378456234931946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656200934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319260597229004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656214736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390298187732697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656228480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335354745388031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656242664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369110226631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656256471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328757256269455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656270029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14000087976455688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656284490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193389773368835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656298477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12873482704162598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656312117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105955719947815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656325792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216131925582886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656339636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580787181854248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656339637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656339637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656423616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.519439160823822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656423617, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656423617, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656437489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337026059627533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656451376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1230003610253334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656465383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13062839210033417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656479611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046808540821075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656493584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519425690174103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656508183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522779941558838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656522379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522547483444214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656537064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12389795482158661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656551475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092230260372162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656565514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108700513839722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656580103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037510216236115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656593938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312530517578125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656607844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088789582252502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301867812871933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656621587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656703970, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5237941741943359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656703971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656703971, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656717758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782340288162231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656731699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229811191558838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656745486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336624026298523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656759186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037289679050446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656772841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272742748260498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656786429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323188841342926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656800507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567706942558289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656814558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283924758434296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656828498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311909258365631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656842572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388363391160965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656856623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371853530406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656870164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375824213027954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656884635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202527165412903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656898780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334956765174866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656898781, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656898782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656981020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5296569466590881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656981021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656981022, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656994985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127672672271729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657008912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349802315235138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657022904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378108412027359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657036873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306479156017303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657050674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12724143266677856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657064721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935931980609894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657078792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12585681676864624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657092752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12395190447568893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657107023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597726821899414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657121448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717271387577057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657135760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13473482429981232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657150735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712106108665466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657164985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059158623218536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657178721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354076653718948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657178722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657178722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657261561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5380802750587463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657261562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657261562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657275729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240675628185272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657289542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265280425548553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657303372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338113248348236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657317259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379699409008026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657331047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363019496202469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657345146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313190460205078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657359175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272008717060089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657373291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506227731704712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657387112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1245088130235672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657401349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035312294960022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657415475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323871612548828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657429756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13407666981220245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657443980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14030081033706665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657458184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361282467842102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657458184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657458185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657541249, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5505567193031311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657541250, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657541250, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657556008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631132245063782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657570250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159041106700897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657584129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250501453876495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657598240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882331550121307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657612992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002412021160126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657627882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840016722679138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657642315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482104241847992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657656880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693153858184814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657671442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485115766525269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657686048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069741427898407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657700628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14041316509246826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657715321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479438424110413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657729883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885114550590515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657744579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333363652229309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657744579, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657744579, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657828553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5647387504577637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657828554, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657828555, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657843653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14193421602249146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657858252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241635262966156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657873407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345593214035034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657888109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304544061422348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657903155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533078134059906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657917959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350869715213776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657932259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337253302335739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657946671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557466864585876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657960905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130796417593956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657975458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248062133789062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657989879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305050134658813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658004344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698026537895203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658018436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622698187828064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658032825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712841272354126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658032826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658032826, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658115633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5804768800735474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658115635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658115635, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658129893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148318231105804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658144466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537278771400452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658158823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402807712554932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658173654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217994570732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658187851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13688704371452332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658202069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949129939079285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658217014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345573842525482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658231325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123517245054245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658245602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047495484352112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658259772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122744858264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658273940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103258609771729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658288030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13115911185741425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658302371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12843884527683258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658316566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060179352760315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658316566, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658316567, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658401394, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5932435989379883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658401395, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658401395, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658415352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216440200805664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658429604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267280876636505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658443666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881459295749664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658457912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309982419013977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658472520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315208524465561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658486782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12149475514888763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658501468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299738585948944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658515968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317678689956665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658530375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656359910964966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658544423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12792307138442993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658559129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254986703395844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658573158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11760612577199936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658587409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106890231370926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658601698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249297708272934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658601699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658601699, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658685092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6059479117393494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658685093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658685093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658699164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12090276181697845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658713166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12745165824890137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658727529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128008171916008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658741728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148799538612366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658755888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12110266834497452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658770105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432499766349792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658784276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870335578918457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658798285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12124145776033401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658812613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12680023908615112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658827115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12703518569469452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658841565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775732785463333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658856329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084383308887482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658870551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267264038324356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658885009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128393292427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658885010, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658885010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658969457, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6175912618637085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658969458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658969458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658983300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751346826553345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658997977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841747164726257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659012254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.121140256524086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659026733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12104526162147522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659040828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1233300194144249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659054858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560363113880157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659068796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11566450446844101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659083021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248659133911133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659097178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12464210391044617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659111130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12339713424444199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659124823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11511978507041931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659138731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12328635901212692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659152640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919079929590225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659166870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1243671402335167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659166871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659166871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659251258, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6279959678649902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659251259, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659251259, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659265719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644700169563293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659280088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399606615304947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659294230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12831485271453857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659308865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12139315903186798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659323399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12365765869617462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659337679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577653139829636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659351507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11953535676002502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659366020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12149475514888763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659380363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1209372878074646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659394788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249811202287674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659409268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12007308006286621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659423531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12104984372854233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659437748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418225407600403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659452246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681370437145233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659452247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659452248, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659535482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6421580910682678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659535483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659535483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659549842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11869238317012787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659564206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134573817253113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659578504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963888466358185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659592586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282549053430557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659606757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841432213783264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659620945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12247999012470245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659635369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573781073093414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659649987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12028391659259796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659664389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206672549247742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659679108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194593608379364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659693624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222472041845322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659707967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287707835435867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659722172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855568200349808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659736572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581901460886002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659736573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659736573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659819616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6618416905403137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659819617, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659819617, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659834095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12027944624423981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659848784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11959521472454071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659863506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12367334961891174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659878209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564717650413513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659892913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447127163410187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659907307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782326549291611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659921895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11950752884149551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659936440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823312193155289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659950833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085125058889389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659965165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781273782253265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659979528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11551768332719803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659993550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627687513828278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660007697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310724914073944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660022348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12504491209983826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660022348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660022349, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660104886, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6829200387001038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660104887, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660104887, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660119600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393999099731445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660134152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676191538572311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660148593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656732857227325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660163548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12193623185157776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660178287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985244810581207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660192921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163397133350372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660206978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310764402151108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660221640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906417667865753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660236176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111731618642807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660251249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594659835100174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660265998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123603954911232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660280604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539462953805923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660295482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272854208946228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660310259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196786165237427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660310259, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660310260, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660395039, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7308523654937744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660395040, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660395040, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660409961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601223587989807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660424544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135559156537056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660439145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725805699825287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660453972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294300109148026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660468623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1190137043595314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660483414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181623786687851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660498550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12083406001329422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660513567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12056971341371536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660528368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509037017822266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660543741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283241212368011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660558600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557026207447052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660573250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603665351867676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660587835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12466657906770706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660602865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617149412631989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660602866, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660602866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660687418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7491692304611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660687419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660687419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660702020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12206432968378067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660716810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11982233077287674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660731440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344163864850998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660746274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449598520994186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660760916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875038802623749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660775583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564753949642181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660790343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11837427318096161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660804780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097416281700134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660819479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908593237400055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660834170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807961970567703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660849223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12310436367988586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660864307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719393938779831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660879087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201542317867279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660894353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12001089006662369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660894354, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660894354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660977426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7533538341522217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660977427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660977427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660992437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193796768784523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661007074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1187913790345192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661021768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515774577856064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661036863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561524868011475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661051585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322309285402298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661066237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033594161272049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661080773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11947090178728104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661095283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537747085094452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661110011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459444671869278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661124539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294073075056076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661138945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668264329433441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661153546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283215135335922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661168208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098276823759079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661182827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500217765569687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661182868, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661182868, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661266432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7561971545219421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661266434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661266434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661281054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416693240404129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661296003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657726973295212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661312332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506704986095428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661326630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082287132740021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661341303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746338218450546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661356145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11513837426900864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661370688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293015629053116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661385178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375156790018082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661399947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161278560757637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661414572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111349917948246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661429041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072828084230423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661443367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323763430118561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661457604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630354285240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661472069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487168282270432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661472071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661472071, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661555575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.759394109249115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661555576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661555576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661569903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640595853328705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661584295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134401485323906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661598684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09646406769752502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661613120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509346425533295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661627592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196281760931015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661641934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483013838529587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661656365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08917921036481857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661670744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086266040802002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661685351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077331215143204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661700148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782227456569672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661715019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692702233791351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661729391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001412034034729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661744336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004107654094696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661759291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107075534760952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661759292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661759292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661842570, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7607152462005615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661842572, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661842572, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661857309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485830694437027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661872020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230829358100891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661886132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488305240869522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661901010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570260137319565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661915146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977969527244568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661929560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219839215278625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661943655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649001598358154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661957829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845445096492767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661971784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09778342396020889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661986105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890550911426544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662000288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502446442842484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662014406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672957450151443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662028480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878538340330124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662042671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299736052751541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662042671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662042672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662125234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.762943685054779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662125234, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662125234, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662139584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044498533010483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662153698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857896715402603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662168139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321275144815445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662182657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662467032670975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662197367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090771853923798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662212154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320265382528305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662226812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029469430446625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662241590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586708784103394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733276605606079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662270951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572990775108337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662285638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344972997903824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662299786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09533485770225525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662314757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834638983011246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662328937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736040025949478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662328937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662328937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662411848, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7638337016105652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662411849, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662411849, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662426109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09244950115680695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662440749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431644320487976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662455205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947254300117493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662470003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873170405626297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662484432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968795418739319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662499183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131758242845535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662513816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09756426513195038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662528174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116018891334534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662542791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473248362541199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662557276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433872044086456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662571812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760371387004852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662585989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474761575460434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662600360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09752921760082245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662615034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608375817537308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662615034, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662615035, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662698638, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.76540607213974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662698639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662698639, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662713567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444135963916779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662728303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388924181461334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662742678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531295090913773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662757219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373640805482864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662771819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09929756075143814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662786329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957404434680939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662801185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509589314460754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662815373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09703259915113449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662830235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774241387844086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662844987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828062146902084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662859613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725950241088867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662874497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502257734537125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662889235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630140244960785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662903899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501485854387283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662903900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662903901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662987633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664561867713928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662987634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662987634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663002227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005549505352974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663016911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609774082899094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663031199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803630948066711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663045767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208433121442795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663060090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09120471775531769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663074727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777296334505081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663088900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348690301179886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663103285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713324695825577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663117713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852944850921631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663132077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968733042478561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663146929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323514580726624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663161695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001974552869797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663176399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943035781383514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663190732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215957999229431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663190733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663190733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663275145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673767805099487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663275146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663275146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663289715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934483259916306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663304167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165052115917206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663318506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021225154399872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663333119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09120970964431763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663347358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144377738237381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663361581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656873881816864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663376818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090672761201859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663391460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197463423013687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663406492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1220879927277565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663421171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518863052129745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663435917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140458285808563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663450871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12279821932315826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663465669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10005267709493637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663480394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114545911550522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663480394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663480395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663565002, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687292098999023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663565003, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663565004, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663579754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10420405119657516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663594567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11859744042158127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663609055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921478271484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663624068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391142964363098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663638823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09649574756622314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663653662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289657652378082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663668677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642848908901215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663683644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684917122125626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663698460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569753289222717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663712946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986719489097595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663727916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12494774907827377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663742790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787850618362427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663757538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531612277030945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663772201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763051360845566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663772201, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663772202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663859175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698436379432678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663859176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663859176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663873926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431890726089478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663888549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469495296478271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663903079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084362119436264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663917796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175481304526329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663932336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114722840487957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663946954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11921758204698563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663961764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736876308918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663976281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209886312484741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663990823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620990931987762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664005728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014153808355331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664020302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081041544675827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664034911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887960344552994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664049406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11530867218971252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664064290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461014300584793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664064291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664064291, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664148312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704995274543762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664148313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664148314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664163394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12025100737810135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664178364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731568932533264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664192910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851902514696121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664208018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757756233215332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664222871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046609282493591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664237760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099366545677185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664252541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354880779981613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664267214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804223269224167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664281966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604243725538254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664296244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039248317480087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664311057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440068691968918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664325477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674947708845139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664340058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417348146438599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664354483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10114539414644241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664354484, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664354484, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664439853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697924375534058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664439853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664439854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664454382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992652922868729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664468964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132226347923279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664483551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09622229635715485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664498174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847452282905579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664512768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565735936164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664527476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446178168058395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664542180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452935099601746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664556957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475940257310867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664571581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156097263097763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664586501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043803930282593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664601511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696745455265045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664616430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10134115815162659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664631686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063081234693527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664646585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11999626457691193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664646586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664646586, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664730769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705788016319275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664730770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664730770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664746128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058726161718369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664761183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09675180166959763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664775979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500451922416687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664790955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316867381334305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664805508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858456790447235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664820231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466189682483673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664834619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062064990401268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664849092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229003220796585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664863761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451152175664902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664878363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801547765731812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664892816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545183718204498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664907368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486061662435532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664921679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666225850582123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664936120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230589658021927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664936121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664936121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665020702, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770980954170227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665020703, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665020704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665035115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09974232316017151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665049666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09374117106199265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665064111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09575874358415604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970461443066597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665092988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693384170532227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665107741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09537187218666077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665122412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056593269109726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665136928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09672918915748596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665151673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297764837741852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665166290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841596871614456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665181213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542818158864975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665196109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246261209249496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665210710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464779824018478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665226231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120420694351196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665226232, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665226232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665311225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714695930480957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665311226, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665311227, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665326008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09842654317617416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665340892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069764792919159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665356156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100892499089241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665370801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099475085735321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665385709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041195899248123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665400355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846994072198868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665415383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087469756603241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665430360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660597681999207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665445635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287773609161377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665460449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164184123277664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665475396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046999901533127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665490220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244188457727432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665505074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882644355297089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665519850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454056948423386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665519851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665519851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665603701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717136144638062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665603702, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665603703, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665618400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980725288391113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665633240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723134130239487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665648002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09088263660669327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665662869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277314484119415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665677950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065533384680748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665692700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835027694702148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665707623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045555025339127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665722606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029876172542572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665737646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10035520792007446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665752628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043168231844902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665767687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272707790136337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665782458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759923607110977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665797797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529007017612457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665813136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911634564399719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665813137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665813138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665896874, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721043825149536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665896875, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665896875, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665911769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374972969293594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665926679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289891809225082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665941432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387485474348068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665955708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281788557767868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665970562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757305473089218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665984681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155656933784485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665999016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846553206443787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666013845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210607945919037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666028633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117379367351532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666043407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463415831327438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666057913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394962131977081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666072420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241670906543732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666087069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366848111152649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666101490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643181949853897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666101490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666101491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666186627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772265613079071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666186628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666186628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666201032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833737254142761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666215504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183437168598175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666229660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401518642902374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666243982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120435804128647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666258562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513056814670563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666272716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126818135380745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666287236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155731439590454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666301543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051828414201736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666316122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340554803609848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666330865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679840296506882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666345276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103239044547081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666359781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050005704164505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666374637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515744984149933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666389304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904467314481735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666389305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666389305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666472846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722175121307373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666472847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666472847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666487204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194999724626541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666501505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09972742199897766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666515828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591227561235428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666529802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063723474740982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666545199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819052904844284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666560003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917039215564728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666575112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791371762752533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666590461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577954143285751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666605376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423498392105103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666620141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076266527175903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666635035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416564345359802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666649920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284483432769775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666664702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512460768222809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666679561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713998973369598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666679562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666679563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666761308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726176977157593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666761309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666761309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666776273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183430463075638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666791198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064389556646347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666806317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862462967634201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666821222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519406944513321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666835861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09421591460704803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666850988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257577687501907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666865644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132579505443573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666880342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770556330680847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666895537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392492264509201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666910802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357531905174255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666925710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938117653131485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666940884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407408118247986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666956073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579852014780045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666971288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310456156730652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666971289, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666971290, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667052054, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730396389961243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667052055, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667052055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667067199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389942467212677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667081889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891831666231155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667096484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962451994419098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667111446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722047835588455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667125872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170675814151764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667139937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825762152671814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667155376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.118283711373806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667170118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924483090639114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667184781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643178761005402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667199759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12265549600124359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667214298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709595680236816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667228827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061666160821915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667243433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460398882627487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667257963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017925292253494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667257964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667257964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667339559, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733845114707947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667339560, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667339560, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667354184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346598714590073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667369187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096586212515831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667383908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077022552490234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667398782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669764131307602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667413420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410945117473602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667428130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11372289806604385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667442791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069139838218689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667457528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11483991891145706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667472436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283750832080841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667487467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11947200447320938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667502480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679788887500763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667517129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181703731417656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667531854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732700675725937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667546813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1196020096540451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667546813, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667546814, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667629594, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77339768409729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667629595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667629595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667644514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951906234025955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667659237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794027894735336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667673755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12008480727672577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667688038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429235339164734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667702562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785841941833496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667717847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535007506608963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667732595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117597222328186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667747703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669755727052689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667762788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212971806526184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667777517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618355870246887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667792350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390302330255508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667807154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663886368274689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667821942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707159876823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667836985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048993244767189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667836986, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667836986, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667920455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736804485321045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667920456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667920456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667935247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986807197332382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667950401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12042950838804245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667965193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037833243608475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667979927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356047540903091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667994500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592894792556763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668009197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954279452562332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668024106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852224379777908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668039070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007335036993027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668054055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112454853951931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668068871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259818077087402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668083914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997245460748672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668099082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068709194660187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668113988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116097941994667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668128824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905879735946655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668128825, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668128825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668211674, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773500919342041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668211675, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668211675, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668226269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11774756759405136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668241205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079639196395874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668255950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124616861343384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668270550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351888090372086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668285271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567089915275574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668300151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216442286968231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668314755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850219428539276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668329327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768789052963257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668343953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824030846357346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668358514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231804847717285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668373235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587631911039352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668387656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047225221991539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668402323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003842204809189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668417183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598428338766098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668417184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668417184, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668499417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773408055305481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668499418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668499419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668513893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490625143051147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668528503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553254932165146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668542811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095150023698807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668557320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09262016415596008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668571731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422418057918549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668585865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428491979837418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668600463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10222814232110977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668615299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237892508506775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668629799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721446573734283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668644552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379359126091003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668659182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927070677280426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668673986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101113259792328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668688761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753492265939713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668703586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753771662712097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668703586, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668703587, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668785122, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738193869590759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668785122, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668785123, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668799695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034836620092392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668814618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313718020915985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668829209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035464704036713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668843768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09959962218999863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668858947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11611591279506683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668873508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030875891447067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668888012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352560132741928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668902952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817839950323105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668917653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793309658765793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668931989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926184803247452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668946527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088120937347412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668961049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926919639110565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668975306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704761743545532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668989639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08692336082458496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668989639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668989640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669072137, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740043997764587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669072138, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669072138, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669086709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09562891721725464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669101232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081914603710175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669115732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756561160087585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669130184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161614418029785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669144725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204233974218369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669159449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09064833074808121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669173985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339982062578201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669188505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09696300327777863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669202980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037474051117897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669217753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640872269868851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669232486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104414165019989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669247188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835058987140656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669261700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643113404512405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669276349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10332643240690231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669276350, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669276350, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669357566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743675112724304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669357567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669357567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669372356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09608607739210129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669387306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569863766431808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669402075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0948735773563385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669416679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271468758583069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669431351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09856298565864563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669445596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085918754339218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669459866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095188707113266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669474964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162882506847382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669489299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900818556547165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669503881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985112190246582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669518964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686750710010529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669533820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046820655465126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669548075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973281413316727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669562393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878913104534149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669562394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669562394, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669644535, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743968367576599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669644535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669644536, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669658773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642104595899582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669673044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451556742191315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669687343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386507958173752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669701702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10025257617235184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669716146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075466126203537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669730552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09613707661628723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669745492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556589066982269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669759877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031847596168518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669774166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086196452379227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669788719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09874508529901505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669802831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09768599271774292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669817219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241729766130447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669831838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09120237827301025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669846441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726030170917511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669846442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669846442, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669930236, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77467942237854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669930237, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669930237, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669944925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951434284448624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669959844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117271542549133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669974315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940160810947418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669989108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006274074316025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670003818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070127993822098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670018631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214022547006607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670033221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279761999845505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670047746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566247254610062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670062447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178618133068085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670076772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270712316036224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670091600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145214200019836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670106038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827276855707169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670120427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616626590490341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670135059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674960911273956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670135060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670135060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670220111, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749477028846741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670220112, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670220112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670234811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690705478191376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670249436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863538295030594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670263873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070742979645729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670278626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070944219827652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670293354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256865620613098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670307685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754718631505966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670322182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097910575568676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670337052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09862445294857025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670351688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522571951150894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670366133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104313425719738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670380779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324126482009888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670395442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104024350643158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670410161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687801986932755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670424898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10030284523963928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670424899, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670424899, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670509146, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750598788261414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670509147, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670509147, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670523503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034000962972641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670537874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573933273553848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670552474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09497185796499252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670567398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885570406913757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670581928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237310081720352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670596890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023505240678787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670611486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695992410182953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670626252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964440554380417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670641064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089567020535469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670655819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023632645606995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670670702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271842777729034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670685146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924427419900894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670699933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250100284814835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670714345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111122369766235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670714345, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670714346, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670799444, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751511931419373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670799446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670799446, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670813723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056293249130249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670828791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510219633579254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670843194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649427026510239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670857898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111558973789215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670872987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641635954380035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670887812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926701128482819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670902279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076853945851326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670916876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738931059837341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670931306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748692601919174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670945620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360351204872131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670960098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822132229804993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670974325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358644276857376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670989024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653557628393173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671003394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09534375369548798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671003442, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671003443, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671084957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756553888320923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671084958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671084958, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671099396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997256189584732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671113781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536875575780869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671128258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293678194284439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671142845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043795794248581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671157433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083914265036583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671171903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234198734164238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671186388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823573917150497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671200960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09883608669042587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671215727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1122567355632782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671230204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211643368005753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671244944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475453525781631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671259745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09914160519838333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671274650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662434995174408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671289445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10459683835506439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671289495, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671289495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671375371, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775739848613739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671375372, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671375372, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671390127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09802483022212982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671404881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855715185403824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671419378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313686519861221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671434369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771392077207565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671449068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11426731199026108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671463363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105821281671524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671478434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790861397981644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671493052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12056625634431839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671507721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496387630701065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671522787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835476219654083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671537816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109901964664459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671552576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084402352571487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671567394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280660331249237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671582311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211161315441132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671582311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671582312, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671667050, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760637998580933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671667051, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671667051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671681902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773083567619324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671696767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036955565214157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671711834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461295187473297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671726771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026841402053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671741748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831058770418167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671756674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864215344190598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671771488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197678208351135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671786223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302895843982697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671800962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12099143117666245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671815887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958773642778397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671830602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842838138341904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671845799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047552302479744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671860944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103840172290802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671875872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870501399040222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671875873, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671875873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671961201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763190865516663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671961202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671961202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671975957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413625627756119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671990853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940961331129074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672005840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425187438726425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672021009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219896376132965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672036051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147034913301468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672050691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844337731599808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672065564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123284175992012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672079936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934049636125565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672095137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062411218881607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672110137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907621145248413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672124981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989834368228912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672140146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965242236852646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672155109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443272441625595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672169818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049510538578033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672169818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672169820, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672252757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766001224517822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672252758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672252759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672267349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304599791765213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672282199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286531537771225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672296931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808128863573074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672311633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241109669208527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672326214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293796449899673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672341071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091360449790955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406953632831573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672371058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298517137765884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672385656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707268863916397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672400252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860412567853928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672415316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12969017028808594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672430160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317678540945053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672445361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325285583734512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672460394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11630431562662125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672460395, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672460395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672543832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767265439033508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672543833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672543833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672558515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566988050937653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672573662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149223893880844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672588793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12232346087694168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672604012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694972425699234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672619192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092289462685585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672634123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782106220722198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672649153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228401958942413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672664038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293987184762955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672679217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339007526636124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672693837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772049218416214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672708593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841769725084305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672723513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138839274644852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672738688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09259173274040222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672753382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372539609670639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672753382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672753383, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672835087, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768469452857971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672835088, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672835088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672849880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178933292627335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672864405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09725296497344971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672878769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476772278547287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672893187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826599597930908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672907558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977867245674133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672922408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395795106887817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672936974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770581662654877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672951647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322201043367386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672966350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989649057388306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672981234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0950709655880928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672995969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742160886526108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673010541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09703372418880463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673024770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261232405900955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673039204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102732479572296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673039205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673039205, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673120962, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768898606300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673120963, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673120964, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673135477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340649634599686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673150142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309004038572311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673164676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425867885351181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673179195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700611025094986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673193791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225681960582733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673208639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977797418832779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673223649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943516343832016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673238437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521243512630463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673253190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714945197105408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673267710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625135898590088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673282536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444574803113937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673296901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813320428133011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673311201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628252476453781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673326276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306241154670715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673326320, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673326321, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673407100, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770264744758606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673407101, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673407102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673421720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085599809885025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673436564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116977110505104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673451731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706361383199692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673466978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725348442792892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673481791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020777583122253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673496843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251869261264801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673511815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09619446843862534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673526590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844972729682922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673541285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023615509271622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673556041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279840975999832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673571132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158883035182953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673585873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363361239433289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673600722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954883903265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673615618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452954471111298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673615668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673615668, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673697519, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771875858306885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673697520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673697521, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673712387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09807215631008148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673727268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079208180308342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673741983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975045919418335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673756601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291707515716553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673771076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09453010559082031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673785939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668230801820755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673801094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411141812801361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673816206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398287326097488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673831518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478890687227249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673846516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183065176010132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673861564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09627819061279297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673876754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170240491628647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673892078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234752833843231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673907033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185886174440384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673907080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673907080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673990266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773709893226624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673990267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673990267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674004923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837927669286728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674020000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743149369955063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674035109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183029621839523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674049549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610140860080719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674064545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006036177277565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674079023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912833362817764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674093526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029467195272446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674108284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09871814399957657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674122975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871776938438416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674137567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900858789682388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674152183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015781968832016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674166805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09472009539604187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674181148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551286697387695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674195472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555104911327362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674195472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674195473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674277412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778046131134033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674277413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674277413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674292044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031537875533104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674306643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138470247387886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674321210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108646750450134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674335876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055607795715332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674350485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031918227672577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674364864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098724067211151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674379131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986310988664627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674393429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164827853441238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674407791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785217583179474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674422097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168984532356262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674436444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09800645709037781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674451160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160330474376678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674465842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11917757242918015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674480489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10914945602416992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674480490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674480490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674562854, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780547142028809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674562855, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674562855, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674577704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569374799728394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674592266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078112035989761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674607278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009168833494186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674622004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087382048368454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674636959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539665073156357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674651767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09911918640136719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674666316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10049830377101898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674681259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059483215212822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674695795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564155131578445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674710156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09415332973003387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674725657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373508185148239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674740379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170957446098328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674755194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09382753819227219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674770171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022344440221786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674770172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674770172, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674853117, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782136797904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674853118, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674853118, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674868239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151231080293655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674883249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346184462308884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674897881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005650982260704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674912662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722338408231735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674927441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322137176990509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674941932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303706675767899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674956337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642563551664352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674970689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845844447612762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674985189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09795655310153961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674999550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972965732216835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675014048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170579493045807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675028340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812899678945541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675043000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232963413000107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675057284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273466259241104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675057284, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675057285, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675138414, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785486578941345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675138415, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675138416, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675152897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200421512126923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675167283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845832526683807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675181722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120946928858757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675196594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09405931830406189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675211381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345330089330673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675226085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767491906881332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675240857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164625957608223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675255368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109949111938477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675270071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544735938310623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675285019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101650670170784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675299882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09775616228580475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675314566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079145222902298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675329357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09884167462587357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675343814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461627691984177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675343815, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675343815, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675426403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778867244720459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675426404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675426404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675441366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021801829338074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675455813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432850569486618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675470049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09197249263525009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675485204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004711389541626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675499873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086505874991417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675514547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100473254919052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675529598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09622912108898163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675544582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388513654470444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675559429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613022744655609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675574002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903017431497574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675588495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043291762471199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675603020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498496890068054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675617462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049318313598633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675632288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387167125940323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675632289, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675632289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675714599, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790331840515137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675714600, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675714600, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675729192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969353467226028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675744110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531973838806152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675768618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431502759456635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675783434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833665728569031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675801587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212124139070511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675816448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09732409566640854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675831737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279352962970734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675846214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197809875011444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675861048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132557153701782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675875923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09514189511537552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675890744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049627959728241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675905930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126299500465393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675921049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10195273160934448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675936089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11835530400276184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675936089, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675936090, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676019818, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793129086494446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676019819, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676019819, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676035038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039306432008743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676050181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09702438861131668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676065130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057538390159607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676080465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203397065401077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676096048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815813183784485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676111372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09657103568315506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676126719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608436912298203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676141896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078692436218262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676157397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105298459529877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676172481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478632897138596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676187013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666177421808243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676202390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044885963201523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676217413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952165186405182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676232739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145361140370369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676232739, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676232740, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676316471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798336148262024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676316472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676316472, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676331904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244729161262512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676347441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101854592561722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676362869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639326274394989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676378180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034976482391357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676393496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603467166423798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676408929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122153908014297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676424435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11838071048259735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676439526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405821889638901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676454862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582417249679565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676470288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715403407812119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676485794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720597207546234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676501011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674495995044708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676516341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068098321557045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676531166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594592034816742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676531167, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676531167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676617632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7798252701759338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676617633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676617633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676632948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189926415681839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676647993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038990318775177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676663063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292354226112366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676678111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577247083187103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676693224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082901656627655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676708097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055714562535286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676723246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485219955444336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676738238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138983964920044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676753752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179085299372673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676769049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535586625337601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676784261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700118333101273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676799457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120888888835907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676814070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795577615499496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676829408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597539484500885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676829409, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676829409, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676911510, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780177116394043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676911511, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676911511, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676926316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127025097608566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676941719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120437830686569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676956792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11515446752309799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676971866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164916515350342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676987233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998716413974762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677002427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784757882356644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677017644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861156344413757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677032804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853639990091324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677048123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005110889673233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677063161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521341115236282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677078032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453376710414886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677093318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008785665035248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677108420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11760544776916504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677123788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297906935214996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677123789, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677123789, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677205810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804377675056458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677205811, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677205811, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677220995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121070608496666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677236369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711140930652618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677251381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129752546548843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677266597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665990203619003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677281790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211231350898743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677297146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673084646463394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677312497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937578439712524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677327628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882090032100677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677342840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856369137763977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677358126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836820304393768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677373470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848177015781403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677388713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131976544857025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677403882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458554327487946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677419221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146172881126404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677419222, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677419223, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677501800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7801834940910339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677501801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677501801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677516542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10552278161048889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677531578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124267429113388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677546489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923377215862274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677561610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637649148702621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677576233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535627603530884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677591009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123763978481293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677606322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588178038597107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677621058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973680019378662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677635831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304163932800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677650396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09333436191082001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677665273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176779642701149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677680121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103669762611389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677694656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979141891002655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677709244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803405940532684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677709245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677709246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677790773, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804396152496338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677790774, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677790774, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677805668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140839964151382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677820821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09241409599781036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677835965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09676700830459595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677850845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940131545066833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677865552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810177028179169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677880304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227597296237946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677895003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09494082629680634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677910299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594635456800461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677924892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582750290632248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677939654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030212789773941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677954544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861717373132706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677969684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994294136762619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677984544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151576787233353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677999144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09851646423339844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677999145, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677999145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678080563, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807923555374146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678080564, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678080564, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678095663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618717223405838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678110257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566718339920044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678125383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377930104732513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678140910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554250538349152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678156386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189348459243774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678171523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087577790021896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678186673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750734061002731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678201647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.095061294734478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678216772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145651638507843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678231808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397583991289139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678246476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355723649263382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678261818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249118506908417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678276337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340496897697449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678291074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142682492733002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678291075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678291076, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678375791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805388569831848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678375792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678375792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678390760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060816258192062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678405625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09915928542613983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678420603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213182866573334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678435203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258708149194717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678450358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926508367061615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678465134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112959146499634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678479758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09692111611366272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678494287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021396726369858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678509130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316716879606247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678523896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815519839525223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678538706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413432121276855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678553484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794682055711746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678568379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172726213932037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678583092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09596018493175507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678583092, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678583093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678667013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805784344673157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678667014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678667014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678682093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09411449730396271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678696742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.099203921854496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678711578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09690281748771667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678726090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020432621240616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678740756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422751307487488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678756120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09561479091644287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678770799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258792340755463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678785736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09796527773141861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678800413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982784628868103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678815251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456255823373795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678829910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09355214238166809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678845011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744638741016388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678860129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520530492067337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678875322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053648591041565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678875323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678875323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678958110, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804100513458252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678958111, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678958111, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678972851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545706003904343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678987716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105311125516891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679002582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534685850143433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679017908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261138528585434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679032562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09620721638202667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679047215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474959015846252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679062364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09592415392398834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679077308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414738744497299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679092350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826737642288208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679107959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08993493020534515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679123206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158028453588486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679138271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531985759735107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679153408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455114394426346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679168610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088247075676918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679168611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679168611, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679250808, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807700037956238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679250809, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679250809, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679265703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084936022758484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679280362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527419298887253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679295138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09807007759809494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679310000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940826147794724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679324953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999804362654686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679339748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09462970495223999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679354566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436948388814926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679369659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266221314668655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679384454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10050204396247864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679399408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060192435979843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679413988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09768807142972946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679428943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999164938926697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679443635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960659593343735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679458471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598740726709366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679458471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679458472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679540605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803234457969666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679540605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679540605, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679555296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176502168178558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679570090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09501972049474716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679585111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981292575597763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679599598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771171748638153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679614418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334828495979309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679628937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08742990344762802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679643761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575804114341736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679658680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09307288378477097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679673596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489991307258606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679688410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09579215198755264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679703208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09497947990894318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679717626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250106453895569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679732421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195578426122665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679747289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983903557062149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679747290, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679747290, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679830861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811052203178406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679830862, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679830862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679845145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09507735073566437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679860393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09555069357156754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679875149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175060480833054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679890058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296501964330673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679905317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357441753149033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679920580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589603334665298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679935856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080113872885704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679950721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563023924827576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679965598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834398865699768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679980626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875458478927612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679995588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085623502731323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680010666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09608373790979385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680025782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09400222450494766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680040877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085687205195427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680040878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680040878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680123341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806923985481262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680123342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680123342, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680138487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09893912822008133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222658560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680153660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486727744340897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680168503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609079152345657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222986240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680183592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09309050440788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680198897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426244884729385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223313920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680214225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981103926897049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680229409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09503877907991409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223641600, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed402905276.log b/recommendation_v4/rcp_logs/gbs_8192/seed402905276.log new file mode 100644 index 000000000..a01994186 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed402905276.log @@ -0,0 +1,1853 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651471631, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651471632, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651491798, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "seed", "value": 402905276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651491802, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651491805, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651491805, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652175033, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652175034, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652175251, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652346394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819746673107147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652358194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13950763642787933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652370534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881269097328186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652383019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904573023319244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652395749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389351636171341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652408616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884398341178894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652421483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907524943351746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652434330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13924768567085266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652447278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884134590625763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652460172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898244500160217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652473283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887031376361847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652486452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890177011489868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652499633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907665014266968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652512573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859662413597107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652512574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652512575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652712534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998720586299896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652712535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652712535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652725388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881732523441315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652738463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890108466148376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652751533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883653283119202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652764206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389695554971695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652777191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872775435447693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652790547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877524435520172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652803510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388952136039734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652816679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868148624897003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652830072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852308690547943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652843267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138639897108078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652856649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841751217842102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652870026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843059539794922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652883342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851898908615112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652896998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871067762374878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652896999, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652896999, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652981062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001035928726196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652981064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652981065, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652994443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828761875629425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653007976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384304314851761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653021414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788056373596191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653034799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137782022356987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653047943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823029398918152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653061246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13844412565231323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653074605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772529363632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653088303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766610622406006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653101702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740934431552887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653114893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382511705160141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653128233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783077895641327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653141345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379188597202301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653155189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796213269233704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653168672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379639208316803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653168672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653168673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653252286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5004276633262634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653252287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653252287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653265578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708610832691193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653279184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772368431091309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653292698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697530329227448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653306612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769014179706573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653320071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768187165260315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653333823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784800469875336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653347472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370435655117035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653361114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13765472173690796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653375078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371624618768692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653389039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387886106967926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653402942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365799903869629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653416595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785314559936523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653430396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601510226726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653444181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698939979076385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653444182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653444182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653526759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008711218833923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653526763, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653526763, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653540242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654515147209167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653553829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361364722251892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653567202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382061243057251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653580886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479453325271606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653594756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694606721401215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653608398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389073312282562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653621948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586211204528809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653635197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551540672779083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653648748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769683241844177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653662141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548630475997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653675591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14583265781402588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653689357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555815815925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653702856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438664376735687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653716252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378909796476364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653716253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653716253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653798585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014587044715881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653798586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653798586, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653812049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539038598537445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653825475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13949958980083466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653838933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617518544197083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653852350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336449533700943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653865867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524192571640015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653879588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13918328285217285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653893244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358157873153687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653906793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335493206977844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653920395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352203369140625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653933699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162244856357574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653947182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13143546879291534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653960415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496491312980652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653973860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324758529663086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653987515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876773416996002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653987515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653987516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654069614, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5021482706069946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654069615, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654069615, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654083292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338728368282318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654096918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358229398727417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654110529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572999835014343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654124101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515248894691467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654137627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758859038352966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654151353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829500436782837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654164727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516052067279816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654178295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12951162457466125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654192069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354657590389252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654206068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258597254753113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654219844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356329500675201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654233652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344124674797058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654247820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544605672359467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654261696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368918001651764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654261697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654261697, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654343980, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5031433701515198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654343981, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654343982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654358008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326826959848404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654371964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337706744670868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654386026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556906580924988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654400442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537903130054474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654414466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13947182893753052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654428330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319000601768494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654442830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530756533145905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654456708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362222909927368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654470665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322514295578003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654484910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196660578250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654499043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562069833278656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654513757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366413116455078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654527965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331632316112518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654542016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060864806175232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654542017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654542017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654625455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5043404698371887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654625456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654625456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654640190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793738186359406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654654280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466113805770874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654668420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545167446136475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654682650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116931915283203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654696554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361607491970062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654710542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341937631368637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654724809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681691884994507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654739073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396196067333221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654753150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276150822639465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654767423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297440528869629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654781389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275618076324463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654795615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289308547973633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654809822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978452444076538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654823648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12469957768917084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654823649, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654823649, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654904855, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5056301355361938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654904856, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654904856, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654919143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114610314369202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654933449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601070642471313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654947941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317933678627014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654962423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029444217681885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654976287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12840522825717926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654990086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12640337646007538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655004104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290772706270218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655017709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244619965553284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655031452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353566199541092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655045231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12494295090436935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655058706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871040403842926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655072242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343991905450821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655086133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100531697273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655100294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291925311088562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655100295, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655100296, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655180511, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072245597839355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655180512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655180512, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655194618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13053689897060394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655208371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12909294664859772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655221821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314363360404968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655236057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382461130619049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655250399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293984055519104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655264023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577994167804718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655277709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12998716533184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655291619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216912791132927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655305163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576748967170715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655319258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719062507152557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655333163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12360979616641998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655347648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1232110932469368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655361776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12561459839344025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655375832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12900619208812714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655375832, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655375833, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655458798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5091200470924377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655458799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655458799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655472522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635362684726715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655486617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658486306667328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655500439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583150506019592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655514244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14007160067558289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655528103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661162555217743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655542016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813841342926025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655555895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276405155658722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655569676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613284587860107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655583389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223848283290863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655597614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409912109375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655611831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721505761146545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655625977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282535344362259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655640096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263185322284698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655653542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145509362220764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655653542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655653543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655735727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5111812353134155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655735728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655735728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655750028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1414056122303009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655764536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14250516891479492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655778475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533349335193634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655792521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393638402223587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655806904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14502838253974915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655820859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409822404384613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655835126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583588063716888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655849350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375603824853897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655863654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13189005851745605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655878144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136182501912117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655892364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779854774475098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655906370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299599289894104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655920993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453057408332825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655935547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647028803825378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655935547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655935548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656017689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5132744312286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656017690, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656017691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656031737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879700005054474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656046293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811707496643066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656060977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177116215229034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656075422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288623094558716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656089867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374565541744232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656104424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13986912369728088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656118837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297416269779205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656133509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1415056735277176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656147564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1408229023218155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656161618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499988615512848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656175540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769140303134918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656189267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12980493903160095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656203150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430774211883545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656217162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12759913504123688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656217164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656217164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656302018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5159565210342407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656302019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656302020, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656315884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322515308856964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656329627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692016899585724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656343381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338574767112732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656356988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320623904466629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656370832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404756784439087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656384597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332983374595642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656398863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336558312177658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656412905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276593387126923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656426766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970325887203217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656441206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177499175071716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656455495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288599669933319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656469575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129357993602753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656483624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220496475696564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656497861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356482058763504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656497862, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656497862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656581070, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5197709798812866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656581072, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656581072, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656595399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370835781097412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656609498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12298467010259628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656623681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305800825357437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656637899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032317161560059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656651948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517138361930847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656666484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542111217975616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656680634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352979987859726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656695183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239435002207756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656709521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100023567676544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656723418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112732768058777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656737662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020358979701996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656751408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312837481498718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656765383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088209927082062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656779139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301681250333786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656779140, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656779140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656862712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5247397422790527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656862713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656862713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656876575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787689805030823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656890481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13223469257354736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656904359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343797624111176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656918122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13021492958068848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656931929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12705978751182556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656945674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190700113773346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656959860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550794124603271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656973927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281068205833435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656987961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116483390331268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657002223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888293504714966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657016392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383561372756958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657029986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375105232000351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657044356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177569210529327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657058561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317392766475677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657058562, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657058563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657142056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5315152406692505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657142057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657142058, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657156074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099122047424316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657170149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501997292041779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657184217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770999014377594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657198368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133268803358078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657212433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726271152496338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657226599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911739945411682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657240789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584739923477173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657254876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393850088119507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657269273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598975539207458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657283754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728754222393036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657298075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450075685977936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657313053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12678854167461395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657327401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088752329349518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657341262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12363197654485703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657341263, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657341263, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657428207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5408981442451477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657428208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657428208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657442378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241338729858398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657456294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243310153484344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657470243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360995054244995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657484184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376221477985382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657497978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604146242141724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657511897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117219507694244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657525687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12716004252433777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657539624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502638041973114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657553343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1245623528957367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657567481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018497824668884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657581514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239744305610657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657595704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374949991703033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657610020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1402326226234436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657624078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596586883068085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657624079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657624079, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657709917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5549882650375366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657709918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657709918, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657724581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643577694892883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657738811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314111202955246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657752525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238941133022308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657766422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885250687599182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657780972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984861433506012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657795681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822031021118164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657809992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456641137599945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657824467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689850270748138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657838816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448373973369598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657853377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039416074752808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657867842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14006178081035614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657882232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444191217422485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657896665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287049800157547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657911127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332336664199829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657911128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657911128, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657998125, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.569856584072113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657998126, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657998126, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658012809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1415831595659256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658027227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325182318687439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658042237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320700824260712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658056751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046975433826447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658071705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497895002365112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658086089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333393007516861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658100391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359671831130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658114871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353781372308731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658129050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13075894117355347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658143466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325116604566574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658157783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13284635543823242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658172090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13673491775989532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658186075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594141602516174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658200334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699877262115479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658200335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658200335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658284650, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5848451852798462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658284651, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658284651, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658298837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125668466091156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658313222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494935631752014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658327359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391411304473877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658342132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171307742595673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658356325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669037818908691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658370427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918339669704437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658384767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320118188858032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658398798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1233672946691513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658413030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13028642535209656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658427138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310386061668396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658441300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306069791316986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658455285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309659481048584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658469342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847399711608887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658483525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030056655406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658483526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658483526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658567947, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5972340703010559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658567948, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658567948, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658581750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12143416702747345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658595819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1263360232114792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658609850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12825143337249756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658623894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301631808280945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658638318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13115853071212769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658652346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12146598845720291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658666734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296994835138321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658681078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159050047397614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658695424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13636577129364014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658709255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275504231452942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658723566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324080377817154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658737399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173725351691246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658751353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12089701741933823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658765470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12487062066793442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658765470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658765471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658851128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.608054518699646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658851129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658851129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658865030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12085188180208206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658878758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12698253989219666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658892656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278155893087387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658906468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131222665309906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658920266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093706429004669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658934160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446144759655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658947919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12859424948692322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658962039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093382328748703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658975970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12644872069358826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658990325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12646286189556122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659004424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786336451768875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659019128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051143288612366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659033500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12679621577262878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659047822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829768657684326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659047823, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659047823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659133403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6174548268318176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659133404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659133404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659147201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666384875774384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659161577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775150895118713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659175567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12130794674158096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659189678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063918262720108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659203572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12339562177658081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659217413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12526148557662964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659231592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158250942826271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659245527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12253908067941666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659259328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12420675158500671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659273087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12291256338357925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659286736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147550493478775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659300550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12235027551651001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659314318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889632254838943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659328274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12471466511487961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659328274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659328275, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659413617, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6245580315589905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659413618, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659413618, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659427761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599375307559967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659442128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11411761492490768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659456225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12897831201553345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659470860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12095210701227188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659485057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12257914990186691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659499298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590501666069031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659513099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11904366314411163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659527539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12100479006767273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659541846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12025170773267746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659556198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222049593925476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659570476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11943966150283813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659584539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12240869551897049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659598676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408193200826645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659613008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690074950456619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659613009, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659613010, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659696179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6322518587112427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659696179, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659696180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659710285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11744926869869232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659724433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110648363828659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659738383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10878518223762512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659752432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296313256025314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659766495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852327197790146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659780690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12239103019237518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659795108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537310481071472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659809770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119859978556633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659823954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157602280378342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659838545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923729628324509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659853015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220603436231613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659867562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294619739055634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659881844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904913395643234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659896132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11613071709871292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659896133, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659896133, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659980859, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6410361528396606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659980860, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659980860, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659994894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12040553241968155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660009270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11967069655656815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660023863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12406820058822632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660038179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104983851313591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660052678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401994526386261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660066819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11802013963460922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660081353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12010172754526138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660095798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922513693571091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660110427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881360620260239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660124766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775386333465576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660139041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659163236618042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660153046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694411188364029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660167249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11343822628259659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660181881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262878179550171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660181881, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660181881, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660266373, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6506502032279968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660266374, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660266374, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660280935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12453753501176834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660295466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678850650787354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660309831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11688544601202011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660324680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227162703871727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660339143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071524769067764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660353609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207699030637741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660367712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274111270904541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660382264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906227678060532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660396946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164949089288712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660412060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11712441593408585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660426921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12462543696165085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660441593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161370798945427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660456451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771467864513397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660471228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271289736032486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660471229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660471229, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660556124, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.661865770816803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660556124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660556125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660571001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277013123035431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660585615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518143117427826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660600196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11863718926906586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660615116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11397581547498703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660629840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12042784690856934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660644739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1197768896818161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660659825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12242291122674942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660674839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12173338979482651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660689626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599969118833542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660705054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445175111293793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660719779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669301986694336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660734411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11892326176166534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660749007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271360218524933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660763904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695407330989838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660763905, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660763906, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660846825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6749802827835083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660846825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660846826, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660861494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12379258126020432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660876385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043952196836472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660890982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516378074884415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660905826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167357787489891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660920343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060626804828644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660934834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787156760692596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660949202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119374580681324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660963360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11368997395038605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660977827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102914810180664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660992165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994520783424377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661006915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12365061044692993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661021839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823149979114532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661036511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12278957664966583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661051661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12069886177778244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661051661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661051662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661134243, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6894315481185913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661134244, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661134245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661149152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12059152871370316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661163653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12069932371377945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661178268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657549440860748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661192796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779584735631943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661207365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447993129491806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661221984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072642058134079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661236371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248382717370987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661250688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542472243309021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661264999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658194869756699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661279284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506518930196762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661305471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911008715629578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661319813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128365620970726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661334019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652246862649918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661348224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848966777324677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661348277, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661348277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661429961, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7075953483581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661429964, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661429964, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661444263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639489442110062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661458914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989250242710114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661473759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822960525751114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661488233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330640316009521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661502788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099964901804924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661517610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11919862031936646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661532044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440346390008926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661546436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11641742289066315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661560784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11806654930114746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661575368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414056271314621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661589662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030544638633728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661603945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615146696567535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661618024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962222516536713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661632297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910722613334656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661632298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661632298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661715524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7282242178916931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661715524, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661715525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661729648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004821211099625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661744039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595746129751205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661758315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918224066495895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661772599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516500473022461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661786734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565835028886795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661800859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774337500333786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661815133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09035241603851318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661829362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113575100898743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661843735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582055896520615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661858210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11801189929246902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661872853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869535058736801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661887294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169269144535065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661902106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046934425830841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661916797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909556597471237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661916798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661916798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662000893, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7468910217285156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662000894, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662000894, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662015503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823681741952896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662030091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294310539960861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662044010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593727231025696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662058571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050170660018921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662072706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144224762916565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662087061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782667994499207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662101162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992449522018433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662115449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109036952257156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662129478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937143325805664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662143863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110287606716156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662158034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643333941698074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662172168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057796329259872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662186321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893215984106064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662200538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113469198346138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662200538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662200539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662284056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7602785229682922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662284057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662284057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662298377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107206642627716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662312451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100597083568573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662326870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046474501490593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662341508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768430680036545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662356427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239002645015717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662371282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347556114196777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662385968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132960975170135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662400811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560515522956848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662415831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810061544179916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662430463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657986253499985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662445044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291825026273727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662459076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09498506039381027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662473786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903867542743683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662488002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800617933273315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662488003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662488003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662571478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7612440586090088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662571478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662571479, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662585633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09196124225854874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662600267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468873381614685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662614695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973555594682693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662629241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902395308017731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662643621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111479997634888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662658143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112132668495178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662672588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09789379686117172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662686796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108769476413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662701160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540840774774551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662715314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519007593393326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662729707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917702317237854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662743763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479922592639923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662757820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09843868762254715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662772226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631629079580307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662772227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662772227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662853049, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7605648636817932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662853050, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662853051, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662867809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453695058822632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662882620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458430647850037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662896986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596440732479095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662911578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423330962657928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662926230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09844663739204407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662940761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006583482027054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662955529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717222094535828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662969653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09656419605016708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662984381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887797921895981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662999127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935727506875992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663013835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685173958539963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663028893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146412119269371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663043613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168329194188118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541256308555603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663058452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663141028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7628855109214783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663141029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663141029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663155548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006113663315773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663170229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157359778881073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663184508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813833773136139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663199209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211723297834396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663213860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09118825197219849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663228701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834829717874527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663243068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449545085430145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663257750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727156698703766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663272064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826419293880463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663286408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038845241069794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663301225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130651980638504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663316058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949540346860886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663331027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092979907989502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663345951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117076426744461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663345952, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663345953, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663427504, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627204060554504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663427505, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663427506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663442252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784751176834106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663456995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173382610082626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663471768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130397021770477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663486524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09191540628671646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663500943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150563582777977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663515429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799773037433624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663530577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110162734985352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663545374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129823327064514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663560526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1217532753944397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663575318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054505705833435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663590107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192307621240616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663605173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12343083322048187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663620203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922470152378082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663635154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093252897262573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663635155, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663635155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663716701, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.763086199760437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663716702, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663716702, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663731563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452700406312943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663746453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11835993081331253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663761181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943161696195602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663776389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439400374889374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663791361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09734898805618286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663806379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279719322919846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663821662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614794492721558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663836756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656874626874924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663851948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11660734564065933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663866867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899064153432846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663882185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12458959966897964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663897445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739947110414505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663912397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720562726259232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663927239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076003909111023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663927240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663927240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664010079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634991407394409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664010079, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664010080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664025050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138763427734375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664039847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554230004549026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664054536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065344512462616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664069496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11790971457958221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664084387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494779586791992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664099322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11930850148200989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664114421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722738087177277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664129072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12184260040521622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664143806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11621535569429398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664158866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105328053236008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664173459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838357359170914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664188250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822886973619461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664202972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494293808937073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664217947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446838825941086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664217948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664217948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664301221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7660683393478394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664301221, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664301222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664316391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976563930511475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664331510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723067820072174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664346257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867631435394287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664361648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806574672460556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664376706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097189784049988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664391748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920815169811249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664406638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403042823076248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664421352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084703579545021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664436253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602352768182755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664451029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017441213130951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664466219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438129305839539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664481088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691792726516724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664495978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474026203155518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664510857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086853802204132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664510858, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664510858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664595448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.767128586769104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664595449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664595449, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664610380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951074957847595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664625501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127587616443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664640499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09667656570672989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664655262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858919471502304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664670097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527600139379501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664684895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502174496650696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664699797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541025549173355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664714683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050359383225441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664729544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217798501253128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664744499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12009777128696442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664759428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172926053404808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664774351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162971913814545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664789608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063369572162628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664804435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12065791338682175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664804435, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664804436, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664888577, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679428458213806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664888578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664888578, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664903808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056744307279587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664918796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686340391635895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664933799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490384697914124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664948789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103971466422081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664963390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834019631147385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664978103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589830577373505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664992729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689693689346313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665007232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1122908815741539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665022053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484613478183746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665036789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754530131816864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665051363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536693036556244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665066133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11505033075809479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665080499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655016452074051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665095017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245692729949951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665095018, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665095018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665179318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687231302261353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665179319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665179319, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665193943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991183131933212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665208782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09460242837667465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665223197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09629514068365097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665237828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09675031155347824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665252294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738292545080185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665267048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09517312794923782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665281606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10025905072689056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665296320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09699917584657669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665311126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336016118526459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665325828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850762575864792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665340955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484777390956879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665355814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297292470932007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665370660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428475379943848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665386049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114716529846191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665386051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665386051, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665469353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702964544296265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665469354, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665469354, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665484206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832131862640381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665498766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774210095405579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665513508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011657118797302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665527938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115676373243332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665542400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100670158863068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665557233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757340490818024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665571640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105553805828094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665586270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643874108791351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665601240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284318774938583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665615735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151012986898422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665630346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002276301383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665644911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282547026872635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665659540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003781855106354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665673917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496444791555405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665673917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665673918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665757475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7697357535362244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665757476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665757476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665772110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008140444755554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665786642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694951564073563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665801210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09141485393047333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665815684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335971415042877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665830468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670991986989975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665845061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079997643828392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665859587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039476841688156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665874368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432478785514832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665889213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001417338848114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665904151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430619865655899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665919115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314653813838959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665933772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839755833148956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665948709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596249252557755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665963681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868170112371445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665963682, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665963682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666047362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708514928817749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666047363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666047363, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666062151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298645496368408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666076869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416081547737122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666091287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455246269702911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666105420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286732017993927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666120142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737911611795425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666134467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10128167271614075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666148886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885524958372116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666163497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238613933324814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666177907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11231443285942078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666192458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529325902462006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666206835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377984493970871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666221046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271511226892471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666235539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448960214853287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666249819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699984431266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666249820, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666249820, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666335143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712752819061279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666335144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666335144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666349649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846805572509766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666364336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216074436903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666378674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432850569486618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666393166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138452053070068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666407712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527149587869644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666421764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265778541564941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666436058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155604034662247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666450201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043548583984375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666464510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317460983991623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666479243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630780458450317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666493592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336211323738098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666507909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465807467699051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666522631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051015555858612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666537194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949947893619537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666537195, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666537195, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666620421, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719082236289978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666620421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666620422, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666635137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024491935968399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666649689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10005109757184982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666664202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065589040517807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666678316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089564323425293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666693248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835602134466171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666707571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961975902318954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666722270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713335871696472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666737194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553364992141724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666751799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391910165548325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666766378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102958232164383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666780822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386902093887329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666795294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323270410299301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666809798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534048825502396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666824253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750764608383179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666824254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666824254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666907270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77168208360672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666907270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666907271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666921809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267499834299088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666936234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076992213726044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666950947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10902368277311325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666965408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553031414747238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666979772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09474509954452515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666994603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131652370095253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667008878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157237201929092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667023230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791938006877899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667037877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328307747840881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667052897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041509360074997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667067361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973578691482544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667082111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438381671905518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667096821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665946453809738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667111461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359269380569458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667111462, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667111462, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667194706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772057831287384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667194707, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667194707, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667209835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11343999952077866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667224329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933522880077362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667238756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963496565818787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667253387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072702407836914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667267477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115323215723038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667281554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783040523529053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667296524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11803048104047775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667311298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916643589735031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667326093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696617305278778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667341187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236400306224823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667356002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803011059761047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667370809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052369326353073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667385708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536417156457901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667400330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039696633815765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667400331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667400331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667486055, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714381814002991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667486056, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667486056, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667500896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371860653162003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667515954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998880863189697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667530757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151552200317383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667545499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674116015434265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667560098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392677038908005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667574809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409730464220047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667589518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732229799032211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667604263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502863466739655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667619161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232750117778778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667634079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063301354646683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667649224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593245923519135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667663940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757101118564606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667678858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733812302350998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667693946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935953795909882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667693947, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667693947, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667777577, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7721333503723145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667777578, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667777579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667792713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200442761182785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667807901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831836611032486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667822770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11956508457660675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667837449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393530666828156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667852145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074434146285057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667867147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531837284564972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667881673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189184337854385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667896734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11706958711147308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667911717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359743773937225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667926321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592935234308243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667941077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363410949707031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667955777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577132552862167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667970324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693986505270004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667985245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549811273813248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667985246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667985246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668068849, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729451656341553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668068850, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668068850, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668083729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105070412158966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668098830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12152128666639328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668113511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071529984474182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668128199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361563950777054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668142972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609168350696564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668157789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096084713935852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668172817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830987244844437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668188017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102236658334732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668203333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244923621416092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668218372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300552636384964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668233522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047206819057465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668248831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098240315914154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668263786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213255673646927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668278629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957322269678116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668278629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668278630, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668361201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733933925628662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668361202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668361202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668375989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830538511276245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668390623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099719256162643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668405170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089000105857849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668419856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295223444700241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668434665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620418936014175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668449612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112242192029953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668464165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866345465183258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668478819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868684202432632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668493521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863147884607315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668508237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194140672683716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668522870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057443618774414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668537068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504019260406494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668551580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001214385032654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645406484603882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566234, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668566235, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668649443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736559510231018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668649444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668649444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668663864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11493359506130219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668678448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572365671396255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668693175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075582563877106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668708013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09219758957624435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668722590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382222920656204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668737127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600171238183975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668752083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292893648147583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668767353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266669631004333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668782242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740269720554352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668797411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351830720901489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668812602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961853712797165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668827811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301130056381226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668842954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769873112440109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668857885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745930671691895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668857887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668857887, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668940313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773674726486206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668940314, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668940314, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668955080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009936779737473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668970051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356488078832626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668984750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080040037631989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668999397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968709200620651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669014195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589410156011581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669029117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024806648492813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669044005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391764342784882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669059168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815531015396118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669074176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826365649700165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669088812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877732932567596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669103742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103912964463234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669118593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951891750097275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669133376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713057965040207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669148143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0875130146741867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669148144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669148144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669229830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738063335418701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669229831, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669229831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669244761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09587721526622772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669259644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106683313846588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669274573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858206450939178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669289552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158545523881912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669304495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237496346235275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669319492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09066607803106308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669334275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335546731948853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669348801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09772074967622757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669363306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412737727165222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669378168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692570358514786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669392981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075430363416672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669407805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817232728004456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669422315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640104115009308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669437026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404159128665924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669437026, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669437027, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669520224, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739784121513367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669520225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669520225, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669535116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678374230861664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669550221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574622452259064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669565109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09523353725671768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669580024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268575698137283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669594782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09807257354259491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669609182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036353766918182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669623515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939355194568634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669638382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192069411277771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669652814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090758666396141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669667326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098741963505745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669682364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716075450181961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669697325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504095256328583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669712044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994888097047806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669726869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882140696048737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669726870, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669726870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669807943, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774314284324646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669807944, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669807944, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669822481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738036781549454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669837074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478634387254715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669851669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399822890758514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669866207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10063517838716507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669881009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736851394176483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669895966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09601115435361862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669911297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588589310646057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669925998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411686450242996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669940805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072895675897598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669955771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09908285737037659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669970381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09750610589981079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669985065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283509641885757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669999901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09141449630260468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670014912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786201059818268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670014913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670014914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670096085, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745040655136108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670096086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670096086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670110831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015322268009186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670125797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142719000577927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670140492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09894315898418427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670155577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060698330402374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670170505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175041854381561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670185519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281802713871002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670200226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295681655406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670214810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584192723035812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670229721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145244747400284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670244222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464563757181168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670258986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176695883274078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670273599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877106338739395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670288115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593461245298386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670302858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686276108026505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670302859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670302859, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670384653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745293974876404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670384654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670384654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670399689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717935860157013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670414572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09847749024629593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670429060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739550739526749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670443770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074003353714943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670458445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333920270204544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670472734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737534612417221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670487151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0974242091178894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670501810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09844139218330383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670516283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559909045696259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670530969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488833487033844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670545788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324802249670029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670560392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007905751466751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670574890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704905539751053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670589300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029508173465729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670589301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670589301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670672591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747678756713867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670672592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670672593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670687056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041945293545723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670701488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551738739013672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670715968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09473259747028351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670731046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924458712339401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670745739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220786929130554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670760863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060194879770279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670775510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693423449993134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670790333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980036854743958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670805290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085733100771904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670820211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002650186419487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670835118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282011330127716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670849540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919870644807816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670864525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207056790590286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670879004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119183823466301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670879004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670879005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670963184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751191258430481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670963185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670963185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670977548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581377148628235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670992656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482316464185715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671007290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614874958992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671022032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146502196788788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671037143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672386735677719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671052126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859683156013489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671066784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840944200754166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671081661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819262474775314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671096395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740560293197632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671111230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352522134780884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671126088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844647884368896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671140587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343514382839203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671155713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693550109863281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671170585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09653159230947495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671170635, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671170635, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671252464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750065326690674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671252465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671252465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671267246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994252935051918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671281912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557056218385696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671296601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242240875959396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671311329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477522015571594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671326053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821282863616943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671340593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12420887500047684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671355435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921511054039001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671370275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934014081954956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671385228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308761686086655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671400116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123194471001625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671415146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516707390546799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671430036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989200532436371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671445101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066519021987915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671460036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486659407615662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671460082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671460083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671541830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753995656967163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671541831, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671541831, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671556880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993204116821289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671571771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818881005048752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671586459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227034032344818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671601733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11799252033233643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671616597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420032382011414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671631120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159062385559082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671646256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756858438253403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671661072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12008000910282135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671675898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579030215740204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671691140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913442820310593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671706367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261257529258728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671721234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069950461387634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671736169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349726468324661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671751055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156562715768814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671751056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671751056, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671833950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753637433052063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671833951, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671833951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671848950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716667771339417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671864000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073228716850281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671879062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148456409573555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671894292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047735810279846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671909465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892483592033386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671924706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822834819555283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671939720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018039807677269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671954758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251349747180939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671969873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12091082334518433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671985076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008087545633316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672000043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885624587535858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672015493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511958599090576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672031053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421549528837204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672046422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956082493066788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672046422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672046423, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672128528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775506317615509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672128529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672128529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672143780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11439725011587143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672158981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11915957182645798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672174249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443984508514404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672189683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292374134063721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672205208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109115928411484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672220307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158956378698349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672235691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175887286663055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672250454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940655320882797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672265698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068594455718994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672280830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09890744090080261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672295901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982246696949005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672311513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951825976371765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672326880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470032691955566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672341981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091388016939163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672341981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672341982, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672424804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757564783096313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672424805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672424805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672440172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297285556793213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672455483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945306301116943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672470752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840769112110138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672485807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266057193279266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672500954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371589452028275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672516458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205576360225677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672531877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11415136605501175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672547391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339927464723587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672562617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605619102716446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672577658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849667340517044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672593057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911207973957062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672608157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374388635158539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672623822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327707022428513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672639473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600472033023834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672639473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672639474, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672723184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760140895843506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672723186, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672723186, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672738245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09635359048843384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672753816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157336086034775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672769391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12232904136180878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672785001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067429855465889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672800791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006677150726318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672816305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743748396635056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672831643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229138821363449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672846676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223474144935608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672861677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350295901298523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672876516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771882325410843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672891262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753951966762543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672906309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173088103532791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672921457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09264107048511505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672936195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345718264579773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672936196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672936196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673018527, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776203453540802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673018528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673018528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673033735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143986344337463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673048595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09696684777736664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673063362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11533608287572861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673078034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10791213810443878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673092745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937260091304779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673108087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499086230993271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673123381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833192616701126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673138509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137584000825882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673153638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037632286548615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673168914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0969451367855072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673183954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971379354596138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673199039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738202393054962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673214000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274812579154968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673229063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090919584035873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673229064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673229064, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673312729, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761097550392151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673312730, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673312730, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673327818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336118936538696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673343056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399220138788223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673358255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434095561504364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673373504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075262501835823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673388719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345729440450668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673404126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016651451587677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673419680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999985784292221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673435091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569489747285843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673450165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712305456399918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673464919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603657364845276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673479988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469590127468109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673494794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109395831823349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673509360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583586990833282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673524117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416136473417282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673524166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673524166, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673606867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763838768005371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673606868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673606868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673621478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888488590717316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673636253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170461773872375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673651211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738073289394379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673666398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724639892578125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673681213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068723559379578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673696351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255015641450882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673711311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09616556763648987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673725935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837341099977493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673740562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993192121386528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673755335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250266641378403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673770403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166031122207642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673785240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365241020917892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673800007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995357692241669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673814930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457131266593933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673814973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673814974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673900255, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764325737953186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673900256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673900256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673915105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09813448786735535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673929914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750970989465714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673944797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980528056621552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673959598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390094667673111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673974135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09407398104667664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673989170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746144503355026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674004306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403516888618469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674019159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419614613056183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674034270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542979091405869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674049269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257931590080261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674064435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09714223444461823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674079522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144015938043594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674094437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197132617235184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674108964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10182566195726395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674109014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674109014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674193190, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776740312576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674193191, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674193191, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674208171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828910231590271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674223408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749667137861252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674238412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190513730049133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674252914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657233744859695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674267855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086538642644882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674282726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085042804479599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674297616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043999552726746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674312604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899303317070007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674327468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918073356151581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674342366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094970703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674357056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990138560533524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674371749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09499051421880722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674386176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09631041437387466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674400569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669966787099838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674400570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674400570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674486472, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768571376800537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674486473, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674486473, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674500993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330145806074142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674515408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377554386854172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674529952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219596326351166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674544526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655273497104645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674559046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158392578363419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674573584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075905919075012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674588004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963762551546097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202150791883469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674616797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708962380886078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674631154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221695899963379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674645611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832601249217987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674660437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124105155467987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674675334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955831199884415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674690252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939908772706985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674690253, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674690253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674776288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771997451782227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674776289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674776290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674791237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638860404491425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674805917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144786536693573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674821023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027829557657242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674836022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109374672174454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674850946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635987669229507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674865921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999415278434753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674880760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092324018478394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674895842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630727559328079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674910396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620313882827759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674924824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09481685608625412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674940019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474253445863724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674954710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214683413505554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674969485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09388476610183716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674984488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031144320964813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674984489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674984489, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675070329, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771161198616028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675070330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675070331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675085313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124923825263977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675100163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351790279150009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675114884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995310008525848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675129597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725530236959457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675144378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351251810789108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675159094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035999283194542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675173656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653866827487946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675188187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084151566028595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675203004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903000295162201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675217617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09779775142669678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675232390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176199465990067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675246944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769050031900406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675262050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211338847875595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675276595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295592248439789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675276596, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675276596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675363505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777419924736023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675363506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675363506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675378302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242239385843277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675392947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963595658540726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675407443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253992468118668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675422589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09453985840082169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675437691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047186404466629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675452680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741548985242844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675467529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701241135597229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675482066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100889950990677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675496926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576742142438889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675512055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159158498048782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675526863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09871482849121094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675541502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921669751405716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675556158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907015413045883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675570426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262705385684967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675570427, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675570427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675655569, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775924801826477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675655570, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675655570, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675670712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045119166374207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675685378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467255860567093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675699789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09209749102592468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675714661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004200354218483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675729412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950727015733719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675744335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099233478307724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675765505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0967506691813469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675780514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520383715629578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675796209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633336752653122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675810826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09994109719991684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675825367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475575178861618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675839967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496099293231964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675854438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148913949728012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675869387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567454785108566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675869388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675869389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675954093, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774476408958435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675954094, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675954094, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675968636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968538373708725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675983548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047464981675148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675998123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044725850224495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676013039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888775438070297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676027738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028684675693512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676042689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09767556190490723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676057506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038031280040741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676071984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282400786876678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676086524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10106676071882248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676101362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09612448513507843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676116038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489282011985779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676131245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236950010061264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676146386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023569107055664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676161282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849050223827362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676161283, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676161283, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676245652, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780337333679199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676245654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676245654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676260628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138971358537674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676275304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788148105144501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676290031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055399477481842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676305069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238902062177658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676320134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890029162168503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676334980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09579908102750778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676349947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558819025754929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676364538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171858221292496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676379585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181591659784317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676394359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445522516965866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676408718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642440617084503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676423979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096419095993042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676439007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024363547563553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676454273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11606930196285248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676454274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676454274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676538987, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778628468513489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676538987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676538988, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676554383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121923178434372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676569731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199335008859634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676584817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723741352558136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12139029800891876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676615195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620738357305527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676630500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266728699207306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676645809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11942452937364578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676660780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451617628335953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676675980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586967319250107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676691140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081598699092865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676706425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741422325372696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676721348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626745223999023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676736629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753922909498215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676751478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11551567167043686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676751479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676751479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676835328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784557342529297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676835329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676835329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676850754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212132126092911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676865723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507866740226746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676880832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282167792320251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676896260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594627052545547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676911709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904514789581299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676927045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630740225315094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676942527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629831254482269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676957789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414361000061035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676973544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11805244535207748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676989148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621708631515503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677004819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11746064573526382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677020333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244921386241913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677035352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862593352794647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677050941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541257798671722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677050942, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677050942, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677133366, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784021496772766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677133367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677133367, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677148435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111035406589508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677163661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125673353672028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677178916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498051136732101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677193849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147064179182053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677209101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09993498772382736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677224117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857061296701431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677239124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875458478927612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677254038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686930269002914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677269088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235090553760529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677283889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568232834339142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677298787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520799249410629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677313653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106706410646439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677328456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755956709384918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677343539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391210556030273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677343540, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677343541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677427384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786070704460144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677427385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677427385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677442351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307752877473831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677457353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813210904598236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677472174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112925335764885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677487184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11713911592960358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677502323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181667447090149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677517274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11749300360679626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677532497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966171532869339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677547515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760229080915451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677562699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994407534599304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677577900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983673483133316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677593370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11971066147089005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677608867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277391016483307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677624252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532436519861221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677639804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201044172048569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677639805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677639805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677725013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788670659065247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677725015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677725015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677739835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562842339277267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677754949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115812510251999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677769744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188526600599289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677785032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673261433839798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677799975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627102851867676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677815077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203764587640762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677830765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687017440795898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677845765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071613430976868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677860914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429107189178467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677876064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09468884766101837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677891355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704372614622116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677906707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069263726472855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677921601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09925527125597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677936705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839702188968658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677936705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677936706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678022469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787046432495117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678022470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678022470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678037615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333960503339767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678052956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09368181228637695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678068430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09722970426082611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678083854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860507935285568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678099147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895679146051407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678114643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341352760791779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678129875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09628899395465851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678145439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689026862382889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678160920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741821676492691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678176340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094111204147339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678191712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09897607564926147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678207435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004715621471405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678222894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142079532146454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678238185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09862854331731796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678238186, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678238187, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678321359, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788223624229431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678321360, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678321360, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678337281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628765821456909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678352604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09633145481348038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678367948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520705580711365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678383429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09679976105690002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678399125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313232988119125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678414237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199750751256943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678429587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772772878408432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678444356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961853414773941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678459240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545320600271225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678474182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429949313402176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678488748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385017096996307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678503862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170061886310577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678518686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279906541109085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678533689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160376131534576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678533690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678533690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678615662, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791153788566589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678615663, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678615663, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678630974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202021896839142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678646064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969518333673477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678660980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269606113433838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678675774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273982584476471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678690874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09874886274337769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678705805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159865021705627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678720555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0966375395655632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678735398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886843711137772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678750497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385508835315704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678765463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976136475801468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678780383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474221408367157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678795108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786890238523483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678810148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10179410129785538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678824851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09627371281385422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678824852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678824852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678908316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791991829872131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678908317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678908317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678923293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0946439579129219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678938161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992782711982727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678952741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09746194630861282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678967450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969807416200638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678982237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385003685951233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678997627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959002748131752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679012516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347907245159149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679027683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826178848743439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679042541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098362997174263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679057534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540945082902908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679072557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09291791170835495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679087823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787548869848251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679103128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546589642763138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679118394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113038882613182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679118394, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679118395, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679202321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795173525810242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679202321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679202322, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679217374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495059192180634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679232396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256247222423553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679247245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648944973945618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679262597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348733514547348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679277210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09680640697479248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679291872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053454577922821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679307047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09527099877595901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679321788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144523099064827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679336578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09859102964401245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679351844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09043776988983154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679367002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951069205999374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679382064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545547306537628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679396980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298407822847366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679411817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852208733558655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679411817, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679411818, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679494400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793664336204529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679494401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679494401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679509414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099931061267853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679524253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603466629981995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679539058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09758089482784271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679554023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033950954675674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679569085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060872882604599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679583934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0955408439040184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679599022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038282960653305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679614295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283731669187546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679629194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169358551502228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679644564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904139280319214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679659400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817789494991302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679674600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061819851398468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679689704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000683382153511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679704763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563643276691437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679704764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679704764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679787778, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796338796615601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679787779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679787779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679802913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10074558109045029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679818083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09511298686265945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679833246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845756739377975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679848295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808788239955902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679863632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309040546417236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679878871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08788947761058807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679894272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490954667329788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679909790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09209781885147095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679925155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10377095639705658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679940455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09581772238016129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679955570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09623252600431442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679970626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290946811437607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679985953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184053868055344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680001065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997270867228508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680001065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680001065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680085086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78006911277771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680085087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680085087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680099633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09594202041625977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680114755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09558748453855515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680129852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225993394851685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680144911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294283181428909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680160264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478690266609192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680175662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723139345645905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680191059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782039910554886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680206126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11585014313459396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680221119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916212201118469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680236503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968430548906326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed404908753.log b/recommendation_v4/rcp_logs/gbs_8192/seed404908753.log new file mode 100644 index 000000000..dcb008266 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed404908753.log @@ -0,0 +1,1855 @@ +:::MLLOG {"namespace": "", "time_ms": 1784658279987, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784658279987, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784658301684, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784658301686, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "seed", "value": 404908753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784658301687, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784658301699, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784658301700, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784659224563, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784659224565, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784659225038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784659354829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872191309928894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784659367028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792134821414948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659379683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875937461853027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659392477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384267359972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659405350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385139673948288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659418144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882684707641602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659431116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831759989261627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659444268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911612331867218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659457610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870635628700256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659470720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383618265390396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659484016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852068781852722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659497269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385715752840042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659510833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383802890777588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659524112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994906842708588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659524113, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659524114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659606840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503982663154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659606841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659606841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659619630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382921189069748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659632626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838481903076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659646245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840237259864807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659659368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794779777526855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659672718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780073821544647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659686430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791024684906006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659699412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379682868719101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659712788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778206706047058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659726840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777776062488556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659740133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791829347610474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659754061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809239864349365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659768031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848333060741425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659781598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775137066841125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659795715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803647458553314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659795715, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659795716, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659877587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042181611061096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659877588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659877589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659891139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736827671527863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659904818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382444053888321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659918455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371767371892929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659932043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359771192073822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659945122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779909908771515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659958443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378798484802246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659971809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691961765289307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659985562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674500584602356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659999000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632827997207642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660012326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372164636850357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660025894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685451447963715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660039130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373167335987091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660053119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741059601306915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660066387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372901201248169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660066388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660066388, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660148225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5045158863067627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660148226, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660148226, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660161358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629257678985596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660174832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732671737670898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660188244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624390959739685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660202147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675348460674286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660215610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719294965267181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660229308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746456801891327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660242969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13610924780368805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660256549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13707108795642853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660270673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642749190330505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660284448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876603543758392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660298188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588422536849976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660311734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750389218330383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660325433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347351372241974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660339174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136424258351326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660339175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660339176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660422681, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049307942390442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660422682, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660422682, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660435908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586808741092682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660449236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350637525320053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660462356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754047453403473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660475789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410231471061707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660489482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640016317367554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660502877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327681541442871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660516517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511745631694794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660529912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510721921920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660543371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780146837234497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660556717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13485507667064667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660570178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1490192711353302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660584002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505882024765015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660597447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324595987796783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660611013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755583763122559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660611014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660611014, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660693797, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5054449439048767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660693798, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660693799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660707043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348438560962677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660720362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14014002680778503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660733760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590778410434723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660747086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260875642299652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660760600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344234198331833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660774333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391633301973343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660787891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289353251457214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660801523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320856511592865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660814966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258136808872223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660828088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072340190410614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660841413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303355097770691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660854330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465899229049683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660867578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313757598400116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660881144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896510004997253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660881144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660881145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660962857, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061421990394592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660962857, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660962858, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660976189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133186936378479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660989535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293100893497467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661003220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560815155506134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661016630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501405715942383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661030059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378655731678009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661043529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12655267119407654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661056640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533490896224976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661070046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12842705845832825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661083659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303227722644806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661097481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235242664813995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661111107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331627070903778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661124866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422371447086334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661138823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531669974327087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661152502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315647840499878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661152503, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661152503, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661234541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071825385093689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661234542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661234542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661248424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240845501422882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661262171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348960876464844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661276148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355404108762741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661290430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555969297885895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661304433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399197280406952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661318239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283292949199677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661332906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511496782302856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661346922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333910971879959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661360817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325853168964386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661374800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315787136554718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661388617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13568434119224548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661402918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354401290416718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661416853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333576917648315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661430659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023127615451813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661430659, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661430660, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661512478, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5083341002464294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661512479, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661512480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661527218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378193199634552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661541078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470424711704254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661554964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353430449962616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661569016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130997434258461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661582842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365639746189117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661596807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409505784511566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661610913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370925009250641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661625011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387656211853027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661638897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277670741081238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661652849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296180784702301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661666360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272665113210678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661680147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297715783119202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661693967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946730852127075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661707367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12439506500959396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661707367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661707368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661787800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5096920728683472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661787801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661787801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661801640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310318559408188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661815463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599659502506256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661829665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309383392333984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661843889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019941747188568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661857637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12811873853206635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661871803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12618333101272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661885896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885881960391998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661899370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224554061889648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661913076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553747534751892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661926850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470822781324387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661940370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12859947979450226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661953997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13424187898635864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661967718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310272067785263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661981778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911245226860046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661981779, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784661981779, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662063197, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114042162895203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662063198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662063198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662077345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032324612140656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662091315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877674400806427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662104822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288480043411255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662119180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12364430725574493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662133362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946265935897827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662146748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12581516802310944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662160281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978492677211761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662174039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12164569646120071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662187398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357671171426773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662201302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13727900385856628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662214958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12345343083143234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662229069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299516797065735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662243008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256469190120697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885668873786926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256842, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662256842, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662340423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5133918523788452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662340424, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662340424, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662353794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366121917963028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662368280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650792837142944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662381965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576924800872803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662395570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994424045085907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662409253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654452562332153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662423067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825061917304993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662436676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273626834154129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662450479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258450448513031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662464163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.142131507396698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662478465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14091677963733673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662492762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718595564365387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662506986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852437794208527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662521216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229210674762726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662534748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13174015283584595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662534749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662534749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662618868, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5155647993087769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662618869, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662618869, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662633313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14127767086029053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662647500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14246965944766998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662661447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533765077590942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662675438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13944897055625916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662689761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14498162269592285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662703610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14095790684223175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662717755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12575286626815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662731947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750717043876648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662746214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188594579696655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662760688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604722917079926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776856660842896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662788886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328531801700592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662803215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13423846662044525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662817497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629576563835144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662817498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662817498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662901396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5179113745689392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662901396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662901397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662915317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856752216815948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662929631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13798515498638153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662943926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171716034412384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662958144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295279443264008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662972367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361647725105286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662986883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13964518904685974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663001280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277076184749603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663015755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14118692278862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663029586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14059560000896454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663044030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350722312927246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275438815355301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663072037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12975084781646729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663086056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341870129108429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663100179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750360369682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663100180, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663100180, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663182747, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5210840106010437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663182748, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663182748, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663196721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320725828409195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663210652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678203523159027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663224823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346800208091736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663238911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190612196922302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663253206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340359002351761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663267500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323649764060974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663282214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350513577461243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663296696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271337747573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663310714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994616270065308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663325208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316613256931305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663339063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12874913215637207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663352720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096857070922852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663366372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218221068382263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663380130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575440645217896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663380130, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663380131, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663462282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5254508256912231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663462283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663462283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663476112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317549228668213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663489777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228833794593811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663503416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032668828964233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663517105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052722811698914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663530780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349017471075058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663545007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502797484397888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663558730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534682989120483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663573016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382978945970535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663587056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309158205986023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663600730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092853128910065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663615283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299973577260971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663629234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292357325553894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663643317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13056698441505432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663657150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12983548641204834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663657150, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663657151, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663739615, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5319961309432983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663739616, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663739616, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663753550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376085877418518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663767621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208214938640594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663781535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332598775625229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663795500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302037239074707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663809314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12660741806030273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663823202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323586404323578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663837564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539256155490875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663851908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325187236070633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663866210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13103589415550232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663880614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852651417255402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663894844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342168927192688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663908458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373627483844757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663922820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315673142671585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663936613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287213444709778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663936614, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784663936614, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664018157, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5409263968467712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664018158, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664018158, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664032131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102388381958008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664046109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344655305147171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664060187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137370303273201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664074326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13273020088672638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664088267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12687426805496216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664102211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884823977947235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664116135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12574587762355804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664130025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234988421201706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664144317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580213487148285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664158769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684715330600739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664173153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422074913978577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664187888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12637455761432648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664202311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069403171539307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664216138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12331299483776093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664216138, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664216139, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664300757, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5538215637207031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664300758, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664300758, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664315494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168612122535706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664329680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320989429950714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664343846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332508623600006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664357903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704612851142883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664372043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539302349090576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664386143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307644546031952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664400180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12650282680988312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664414271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13415507972240448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664428163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12374832481145859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664442593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948307394981384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664456784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13155274093151093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664471135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330888718366623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664485423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394653469324112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664499497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541299104690552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664499498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664499499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5712177753448486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583690, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664598508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575008511543274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664613015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046343624591827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664627119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152122497558594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664641356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788340985774994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664656472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12900832295417786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664671482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374169886112213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664685816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404397666454315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664700284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581062853336334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664714763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335882127285004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664729357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970055639743805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664743872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13915173709392548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664758390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335444152355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664772752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12725117802619934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664787355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329387128353119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664787355, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664787356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664872485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5897519588470459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664872486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664872486, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664887237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14040637016296387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664901761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312401443719864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664916736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323026865720749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664931240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861275672912598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664946115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421005010604858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664960984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317714899778366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664975572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13227106630802155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664990202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341848522424698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665004811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12895804643630981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665019694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040904700756073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665034473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313854604959488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665049219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536468148231506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665063563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347709447145462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354997754096985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665078298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163277, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6072400212287903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163277, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163278, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665177920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952592968940735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665192956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367144763469696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665207663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323917806148529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665222663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13017809391021729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665237157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515880703926086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665251658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270749568939209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665266594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314704567193985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665280666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199415996670723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665294933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276252567768097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665309176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809023261070251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665323526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765783071517944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665337488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760548293590546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665351748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584078311920166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665366074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12716840207576752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665366075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665366075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665451183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6233605146408081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665451184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665451184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665465271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11839921027421951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665479619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12305781990289688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665493819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1244233176112175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665508200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999087572097778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665522989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812231481075287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665537488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725426465272903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665552158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12521764636039734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665566619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12825195491313934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665581071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323302537202835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665595001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12409287691116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665610123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12804998457431793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665624457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414527148008347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665638723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11548877507448196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665653052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12060507386922836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665653052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665653053, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665738467, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6491081714630127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665738468, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665738469, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665752719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168200746178627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665766854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12250177562236786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665781111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12271711230278015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665795067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256849467754364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665808996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604680866003036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665823008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928512692451477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665836927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366705387830734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665850805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614133417606354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665864980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12201929092407227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665879377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12083500623703003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665893600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247116327285767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665908376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12671560049057007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665922706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12224479764699936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665936916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12508846819400787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665936916, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784665936917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666021769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6842385530471802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666021770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666021770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666035647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12053387612104416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666050382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358692497015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666064750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11689387261867523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666079077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578401178121567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666093227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970062553882599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666107412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093625962734222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666121471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989678651094437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666135710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11909569799900055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666149663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11946966499090195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666163654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11800671368837357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666177348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977841168642044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666191334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814434826374054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666205407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302707552909851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666219547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861368268728256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666219547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666219548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666304332, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7208166718482971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666304333, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666304333, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666318696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100917309522629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666332902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756219923496246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666346973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11980361491441727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666361485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679232865571976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666375667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11854974925518036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666389888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110692024230957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666403623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283967643976212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666418371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11912018805742264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666432492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497130990028381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666446584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526479035615921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666460622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557655036449432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666474520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714500188827515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666488603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12218542397022247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666502880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346852779388428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666502881, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666502881, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666588566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.74155193567276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666588567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666588567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666602960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406029760837555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666616883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634948313236237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666630744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585951060056686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666644715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100049540400505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666658887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044350117444992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666672789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781815439462662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666687313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155009269714355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666701931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11629744619131088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666716160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566514939069748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666730729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769804358482361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666745211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183296144008636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666759400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898750275373459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666773466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433172434568405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666788058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265647411346436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666788058, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666788059, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666874305, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7526703476905823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666874306, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666874306, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666888328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324287205934525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666902679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155402660369873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666917137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11850082129240036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666931530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098177194595337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666945921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109302192926407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666960075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528945714235306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666974592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11628304421901703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666988959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134243905544281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667003068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433486849069595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667017235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11484570056200027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667031494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989979654550552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667045608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406011879444122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667059651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796429216861725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667074114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1204761490225792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667074114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667074115, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667158506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7559629082679749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667158507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667158507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667172964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889028549194336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667187319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496753245592117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667201527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200999468564987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667216332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546646058559418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667230756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171603411436081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667245313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072123646736145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667259292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937671363353729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667273627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633508116006851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667287913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901352018117905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667302715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421369016170502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667317207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021950632333755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667331544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199455708265305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667346082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12371375411748886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667360686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956120491027832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667360686, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667360687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667442881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7618590593338013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667442882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667442882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667457525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12292811274528503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667471809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058665812015533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667486082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266599595546722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667500738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300022155046463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667514975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406271904706955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667529432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380574852228165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667544153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998632550239563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667558970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1191389411687851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667573520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396753787994385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667588579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048728972673416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667603215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131676658987999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667617736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975128412246704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667632189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215638816356659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667647259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207817494869232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667647260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667647261, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667729013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7633564472198486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667729014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667729015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667743652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696646362543106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667758588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605062335729599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667773470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111247718334198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667788514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232543736696243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667803284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656879097223282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667817970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256082355976105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667832813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518719792366028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667847434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082940474152565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667862422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087239533662796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667877189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754744708538055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667892082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198149412870407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667907086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446488857269287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667921903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186998263001442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667937141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11882232129573822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667937141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784667937141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668018745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.764214813709259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668018746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668018746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668033670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701370030641556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668048206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714618653059006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668062801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11424907296895981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668077782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223562061786652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668092332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078959912061691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668106902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057037323713303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668121346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676743626594543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668135803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583195626735687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668150248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320685803890228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668164708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149935632944107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668178981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546234250068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668193508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216865479946136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668207858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759148001670837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668222174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365044116973877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668222217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668222217, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668306152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7647486925125122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668306153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668306153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668320705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403795540332794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668335450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626691579818726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668350114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355811357498169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668364478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913395136594772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668379105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639641433954239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668393999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350499838590622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668408507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267802864313126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668422862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108456701040268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668437636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156584694981575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668451983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065227538347244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668466376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967102855443954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668480863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125451847910881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668495055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569482296705246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668509472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479975491762161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668509473, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668509473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668593951, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7662606835365295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668593953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668593953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668608181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510695725679398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668622579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323992162942886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668636880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09616216272115707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668651041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041678711771965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668665386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195676773786545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668679527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503135621547699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668693908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08913657069206238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668708211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839783400297165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668722385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079697519540787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668736828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783495545387268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668751437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744921118021011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668765723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115483403205872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668780643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986159950494766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668795476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074487715959549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668795477, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668795477, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668881352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676895260810852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668881353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668881353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668896050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144772619009018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668910701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283175647258759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668924848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545306652784348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668939764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531309247016907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668953842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990929067134857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668968267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092645138502121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668982467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679590702056885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668996895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783237963914871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669011113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09695351868867874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669025497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094551607966423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669039648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472668707370758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669053785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669580101966858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669067830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858882963657379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669082095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300211399793625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669082096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669082096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669168330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692927718162537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669168331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669168331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669182613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098208874464035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669196716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873262792825699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669211212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293906927108765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669225760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530391335487366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669240374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073584109544754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669255113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124073714017868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669269727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041084885597229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669284529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538040846586227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669299303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733634233474731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669313607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510848462581635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669328204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311589181423187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669342186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09500420093536377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669357151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09775573760271072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669371616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776984691619873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669371617, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669371617, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669457594, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707017660140991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669457595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669457595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669471938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09186312556266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669486680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443904250860214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669501253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019538342952728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669515996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09800581634044647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669530471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097890883684158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669545301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118568405508995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669560078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766083210706711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669574579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107911169528961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669589135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522343218326569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669603472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411130636930466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669618079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698511451482773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669632335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551409423351288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669646641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689651429653168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669661388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615560412406921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669661388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669661389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669745903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771099328994751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669745904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669745904, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669760734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409845411777496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669775613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104132279753685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669790107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582209378480911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669804822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431825369596481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669819705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889298677444458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669834198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033106803894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669849101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049722209572792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669863215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09736523777246475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669878003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846318304538727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669892391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900215804576874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669906653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696688830852509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669921174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539065092802048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669935486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696126312017441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669949916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514466255903244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669949917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784669949917, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670033930, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716795206069946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670033931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670033932, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670048144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127688944339752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670062577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594422161579132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670076892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662350803613663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670091606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214688628911972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670106042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.092311792075634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670120665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846533834934235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670135012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318780690431595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670149559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779423266649246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670163795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802856087684631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670178110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126125812530518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670192869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318206042051315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670207483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046210139989853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670222361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032683402299881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670236956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154815554618835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670236956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670236956, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670319362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719902992248535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670319363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670319363, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670333966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903002321720123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670348567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300253331661224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670363003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063723266124725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670377729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0918099656701088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670392104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498288810253143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670406524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747072845697403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670421939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105593740940094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670436621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195390671491623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670451727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12240701913833618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670466522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596273094415665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670481371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076009273529053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670496513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12202608585357666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670511376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082592070102692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670526196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140176653862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670526197, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670526197, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670608592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726359367370605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670608593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670608593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670623321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519932955503464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670638129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828391253948212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670652801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905776917934418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670667939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528876632452011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670682611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09640546888113022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670697379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342189460992813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670712405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650155693292618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670727327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626474767923355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670742411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573249101638794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670757167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004483699798584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670772321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1242324560880661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670787340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704244673252106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670802302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420438438653946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670817128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780567675828934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670817129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670817129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670899731, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730265259742737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670899732, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670899733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670914803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147032156586647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670929494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529593914747238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670944106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121894419193268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670958990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177494153380394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670973699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464901268482208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670988330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878790706396103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671003112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691837549209595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671017529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12089917063713074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671032017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711347848176956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671046773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077819764614105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671061193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843328386545181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671075834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085876002907753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671090348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518041044473648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671105094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150398924946785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671105095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671105095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671186170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731436491012573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671186171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671186171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671201123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11910033226013184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671215916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780452191829681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671230379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835769772529602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671245598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913559794425964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671260448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074277758598328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671275312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060437560081482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671290045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138368472456932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671304712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840590298175812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671319516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622086375951767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671334085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069446265697479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671349142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104729063808918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671363813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.096420057117939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671378377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439933836460114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671393000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012272834777832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671393000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671393001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671474654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729725241661072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671474655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671474655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671489237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082924902439117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671503825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019272729754448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671518267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09736814349889755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671532637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088876873254776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671547083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152411699295044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671561414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044747456908226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671575763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040901467204094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671590201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477704554796219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671604571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234639793634415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671619231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12101320177316666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671633819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175750121474266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671648398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168971121311188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671663100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066631436347961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671677352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12050683796405792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671677352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671677353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671760769, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773280143737793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671760770, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671760770, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671775702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113334372639656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671790494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0964871495962143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671805123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537530481815338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671819802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400043427944183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671834113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799982398748398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671848778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477922111749649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671863256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059030070900917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671877630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229583621025085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671892384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490991175174713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671907033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892148315906525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671921517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554353892803192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671936020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503773927688599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671950261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732710361480713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671964644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254962742328644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671964645, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784671964645, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672047181, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732754349708557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672047182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672047182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672061586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010216385126114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672076142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0948290005326271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672090502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0967169776558876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672104941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09718084335327148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672119084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09712252765893936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672133468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551500529050827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672147613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062872618436813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672161826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09790609031915665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672176301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345061868429184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672190848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832735896110535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672205594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574306547641754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672220009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353170335292816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672234435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516150832176208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672249463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072667688131332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672249464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672249464, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672331904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773652970790863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672331905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672331906, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672346546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886396676301956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672360942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725046694278717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672375677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101347953081131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672390063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226182639598846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672404509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09999532252550125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672419385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865913331508636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672433855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005833745002747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672448331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660487413406372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672463068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288863629102707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672477503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014714315533638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672491979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017254948616028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672506432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285357385873795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672520863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095564141869545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672535137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115663081407547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672535137, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672535138, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672615832, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739480137825012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672615833, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672615833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672630326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010105907917023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672644852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696738958358765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672659331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09171672910451889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672673899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208073258399963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672688462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678946226835251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672702803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084529235959053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672717077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055485159158707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672731588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232897102832794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672746117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115064680576324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672760642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447774082422256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672775355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269561409950256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672789790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757773369550705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672804631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050054058432579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672819517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888843983411789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672819518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672819518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672901964, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739936113357544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672901966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672901966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672916599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367866605520248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672931116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10276588052511215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672945404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417564958333969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672959568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281248390674591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672974783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750377178192139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672989246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143844783306122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673003821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785077720880508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673018743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212890803813934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673033488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179200559854507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673048207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526502132415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673062797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433641821146011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673077249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028667539358139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673091864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413458943367004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673106307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694941133260727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673106308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673106309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673190007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741411328315735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673190008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673190008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673204656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857594758272171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673219432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018170714378357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673233769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401420295238495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673248346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121431946754456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673263053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058625802397728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673277165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11315269768238068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673291721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196031630039215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673306042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094789952039719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673320752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294246464967728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673335698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709269344806671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673350233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321040451526642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673364805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577686876058578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673379528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598510503768921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673394239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985577404499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673394240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673394241, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673476355, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740490436553955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673476356, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673476356, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673491082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137558728456497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673505809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09927301108837128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673520474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603984445333481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673534553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083628445863724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673549566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831145197153091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673564091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920422524213791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673578867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770034044981003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673593974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157749593257904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673608656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374292522668839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673623336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055181384086609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673637905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389499366283417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673652524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303709656000137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673667087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565704852342606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673681397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067570149898529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673681398, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673681398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673765686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742127180099487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673765687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673765687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673780187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171112418174744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673794694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099811851978302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673809324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924949496984482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673823782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044216975569725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673838104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09444723278284073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673852904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245757341384888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673867173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170836001634598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673881467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738861560821533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673896046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336083918809891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673910959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364249348640442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673925523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973650962114334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673940284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417762190103531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673954984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568512976169586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673969736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030217707157135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673969736, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784673969737, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674056479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744092345237732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674056480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674056480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674071487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325210332870483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674086207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900008678436279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674100750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098882257938385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674115497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765545070171356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674129731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101941779255867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674143928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080082356929779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674159517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847283691167831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674174701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969369113445282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674189757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565899103879929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674205089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216958537697792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674220035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720092803239822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674235117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003612726926804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674250227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537057906389236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674265300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062610149383545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674265301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674265301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674350417, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744579911231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674350418, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674350419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674365384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361860483884811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674380411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977119952440262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674395211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056604236364365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674409982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671469569206238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674424741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457728803157806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674439528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342015117406845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674454334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726781189441681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674469198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481605470180511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674484336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294239014387131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674499634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11924897134304047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674514980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10641536861658096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674530088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186642050743103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674545031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732463002204895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674560331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11926592141389847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674560332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674560333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674645227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746560573577881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674645228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674645228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674660137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11926387995481491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674675275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790906846523285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674690196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11953384429216385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674704764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448189616203308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674719513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840631276369095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674734963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549966037273407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674750049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157293617725372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674765255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11658242344856262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674780426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150003969669342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674795235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610838979482651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674810032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279892176389694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674824819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633190721273422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674839525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731670051813126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674854435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517553985118866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674854437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674854437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674941315, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746797800064087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674941316, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674941316, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674956235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985300689935684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674971297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.119237519800663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674986178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097937822341919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675000935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11358553171157837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675015705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616391688585281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675030418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956526547670364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675045243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881289839744568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675060130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987008363008499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675075211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300243437290192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675089968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256417632102966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675105010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017627269029617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675120124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093887686729431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675134991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169843375682831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675150024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906955599784851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675150025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675150026, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675236227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747929096221924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675236228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675236228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675250818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11736836284399033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675265851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047876626253128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675280730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111203134059906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675295393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354144871234894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675310199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581883788108826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675325006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154655367136002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675339420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814142227172852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675353928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799571871757507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675368535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09770813584327698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675383052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172649472951889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675397978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598032921552658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675412416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513197630643845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675427214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040043085813522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675442111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11609349399805069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675442111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675442112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675526952, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748189568519592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675526953, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675526953, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675541673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454084515571594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675556409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561888664960861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675570996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10073310881853104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675585777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09248678386211395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675600530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451594531536102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675615044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327611118555069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675629720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239776968955994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675644792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253315210342407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675659753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076105535030365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675674930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329253226518631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675689929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923326015472412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675705153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107889860868454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675720323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723020881414413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675735267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680096596479416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675735268, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675735269, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675818506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750232815742493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675818507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675818507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675833313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964151471853256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675848243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315757244825363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675862719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034443229436874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675877167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09847546368837357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675892016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618465185165405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675906654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008898168802261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675921194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383674502372742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675936051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833372920751572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675950684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785295814275742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675965015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923995077610016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675979438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103135421872139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675994082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957253187894821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676008422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687325149774551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676022747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08696789294481277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676022747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676022748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676106463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752275466918945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676106464, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676106464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676120953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0964154377579689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676135623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151071637868881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676150236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703825205564499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676164886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10124871879816055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676179543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238882154226303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676194227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09081850200891495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676208702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033264622092247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676223247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971212387084961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676237912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103432297706604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676252621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629823058843613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676267249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10987204313278198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676281980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860294103622437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676296636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702292621135712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676311531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032353937625885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676311532, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676311532, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676393281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751433849334717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676393282, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676393283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676408206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09603924304246902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676423336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554690659046173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676438168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0950503796339035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676453175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257495939731598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676468126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09822263568639755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676482661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138359665870667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676497229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947489738464355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676512209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145499557256699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676526689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900191962718964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676541278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098347082734108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676556234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674287378787994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676571338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041121631860733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676585908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958507657051086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895469039678574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676600552, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676682658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77522873878479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676682659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676682659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676697304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638441145420074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676711999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438171774148941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676726480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351470112800598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676741079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006443411111832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676755878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706165432929993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676770724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0960162803530693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676785896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521396994590759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676800482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102692611515522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676814856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107123643159866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676829488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09869901835918427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676843657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09840186685323715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676858021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240103304386139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676872786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09036347270011902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676887402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070370227098465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676887402, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676887403, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676970771, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753761410713196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676970772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676970772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784676985457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09923937171697617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677000332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014372780919075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677014820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994548425078392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677029682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954989492893219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677044584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072864592075348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677059339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227768123149872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677073981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209447890520096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677088366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529769957065582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677103107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148883610963821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677117415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180774867534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677132541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142150521278381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677147310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075846329331398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677162184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560718178749084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677177177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631829500198364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677177178, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677177178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677258831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775485634803772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677258832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677258832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677273853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670337080955505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677288723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846008569002151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677303479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681327432394028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677318541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614781826734543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677333415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275444388389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677347953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759209096431732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677362716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09760276228189468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677377791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815816581249237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677392740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048969030380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677407625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042083278298378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677422635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231301188468933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677437661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021484434604645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677452647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659989714622498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677467631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012307018041611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677467632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677467633, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677548552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757453918457031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677548553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677548554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677563356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294932872056961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677578033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596641153097153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677592886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09528881311416626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677608194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834437072277069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677623119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194918513298035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677638444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028335452079773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677653559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765327513217926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677668711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979016125202179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677683770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953936725854874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677698853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09941855072975159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677714014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027035340666771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677728731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09865130484104156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677743859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271131038665771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677758567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108668148517609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677758568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677758568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677841955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758045196533203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677841956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677841957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677856499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058601588010788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677871857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053425595164299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677886424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064748466014862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677901186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136309057474136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677916172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678496211767197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677931100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880690813064575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677945666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772278159856796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677960439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09745806455612183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677975114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670184344053268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677989526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343519598245621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678004030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805102437734604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678018229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033305749297142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678033024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647996515035629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678047685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09536559879779816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678047731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678047731, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678130459, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757459282875061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678130460, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678130460, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678144984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995338320732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678159605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597242414951324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678174297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290095955133438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678189266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443823039531708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678204032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840485990047455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678218665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12289216369390488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678233551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788550972938538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678248372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920508414506912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678263388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188022792339325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678278210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139551550149918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678293341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499494314193726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678308381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828504174947739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678323487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066369041800499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678338439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413217544555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678338483, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678338484, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678422183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776190459728241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678422184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678422184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678437277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09714909642934799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678452293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811784327030182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678466953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283499002456665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678481798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11811192333698273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678496331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430195719003677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678510773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113375723361969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678526185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074720099568367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678541296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032898515462875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678556393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470280796289444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678571870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824974626302719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678587272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084116250276566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678602430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079145222902298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678617680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292116343975067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678632854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150255799293518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678632855, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678632856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678713547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763019800186157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678713549, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678713549, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678728757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781121999025345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678743790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042304337024689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678759179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422759294509888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678774483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999657213687897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678789773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796412080526352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678805140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863330215215683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678820391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180072486400604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678835465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265945434570312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678850675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12045430392026901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678865946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938068479299545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678881156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872292518615723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678896619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476361215114594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678912286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351309180259705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678927636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905911773443222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678927637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784678927637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679010367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764267921447754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679010367, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679010368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679025832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444143950939178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679041382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11914826929569244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679056867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431998759508133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679072450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251617223024368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679087995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082107573747635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679103193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925379186868668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679118649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154226958751678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679133486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920005291700363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679149130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107422485947609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679163898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09917393326759338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679178851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099138930439949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679194325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976127535104752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679209506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373005270957947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679224283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036645621061325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679224283, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679224284, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679305654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766224145889282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679305655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679305655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679320583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327454447746277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679335666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276201754808426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679350629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846871882677078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679365521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250832676887512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679380394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223548650741577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679395466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158936470746994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679410648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409647762775421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679425874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11276110261678696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679440789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658910870552063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679455695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856159776449203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679470900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12966232001781464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679485807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320150643587112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679501222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281570792198181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679516483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11655424535274506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679516484, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679516484, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679598706, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766678929328918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679598708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679598708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679613736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09538178890943527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679628926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155154556035995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679644165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12193825095891953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679659411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071677878499031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679674726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097772866487503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679689931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775579512119293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679705008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211900413036346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679719833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341337859630585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679735141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313203930854797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679749899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11807423830032349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679764977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810986906290054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679780156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141656339168549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679795386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09225845336914062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679810191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349839925765991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679810191, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679810192, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679892051, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769864797592163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679892053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679892053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784679907249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011153906583786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679922235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686257690191269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679937016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406899988651276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679951971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076369434595108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679966859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996238887310028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679982078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357410460710526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679997098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773316025733948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680012128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301535367965698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680027192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965254366397858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680042406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09525133669376373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680057573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09714441001415253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680072669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674493223428726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680087630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250845551490784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680102768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121256113052368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680102769, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680102769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680186446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771611213684082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680186447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680186447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680201683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359865427017212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680217129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028703823685646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680232289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039377897977829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680247464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705272853374481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680262732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230817645788193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680278133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09911313652992249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680293700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085686981678009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680309062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526680946350098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680324289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691183805465698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680339476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606282949447632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680354915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440484434366226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680369791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788781940937042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680384489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593172162771225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680399851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130446195602417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680399890, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680399890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680483211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773980498313904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680483212, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680483212, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680497885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821207612752914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680512865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190617084503174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680528141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725502669811249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680543694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695639252662659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680558739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072077810764313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680573953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204949975013733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680589280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09587429463863373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680604029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778123885393143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680618739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962281584739685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680633592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239101946353912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680648792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011098101735115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680663620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284099727869034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680678509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926744550466537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680693598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394667088985443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680693650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680693650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680776455, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775459289550781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680776456, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680776456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680791398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981733500957489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680806190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107809878885746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680820763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996847003698349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680835432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265941172838211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680849971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09381452202796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680864826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632255673408508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680879739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362081974744797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680894529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383623093366623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680909606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488373041152954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680924290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131029576063156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680939207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09540805220603943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680954222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148938536643982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680969178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175429821014404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680983736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183583945035934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680983784, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784680983785, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681066979, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777686715126038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681066979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681066979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681081583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861286729574203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681096570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763608664274216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681111431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209841281175613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681125818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555166751146317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681141054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082439333200455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681156123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888291150331497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681170966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069537162780762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681186113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818210452795029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681201277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918281972408295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681216288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931375622749329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681231187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000932976603508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681246239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09395729750394821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681261028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09448432177305222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681275901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547591000795364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681275902, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681275903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681359137, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780077457427979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681359137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681359138, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681374125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273171961307526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681389158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394757777452469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681404329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003136932849884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681419473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538869351148605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681434624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101079136133194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681449559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10051004588603973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681464555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016517132520676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681479342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167046636343002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681494163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777716338634491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681509028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007218062877655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681523997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09705885499715805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681539267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157678812742233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681554662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11892847716808319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681569909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901457816362381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681569910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681569910, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681654735, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782136797904968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681654736, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681654736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681670115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538846045732498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681685279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006985604763031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681700933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990853816270828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681716123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038463771343231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681731441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503783077001572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681746759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887580573558807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681761863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09994149208068848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681777337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063770055770874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681792204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583483427762985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681806917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09355238080024719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681822488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384513437747955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681837237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151863843202591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681852171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09393402934074402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681867168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10030394792556763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681867169, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681867170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681951850, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778218686580658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681951851, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681951851, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784681966917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100616306066513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681981892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137327030301094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681996802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105447471141815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682011849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675612837076187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682026971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313858836889267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682041925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10222437232732773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682056868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687699913978577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682071816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826552659273148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682086799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976090133190155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682101590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711572527885437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682116608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156520992517471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682131383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742366522550583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682146493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180482268333435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682161236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298536717891693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682161237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682161237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682243903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784460783004761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682243903, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682243904, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682258993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212673991918564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682273734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844377428293228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682288554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172603070735931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682304027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09359940886497498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682319317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293970257043839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682334540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785417258739471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682349918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584458500146866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682365006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089842975139618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682380382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579598695039749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682396061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033281683921814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682411497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09727093577384949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682426738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735788196325302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682442051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823369979858398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682456992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380303859710693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682456993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682456993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682541783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785574197769165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682541784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682541784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682557188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034020036458969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682572150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412675887346268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682586900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09207472205162048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682602441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072078555822372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682617353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866384953260422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682632494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109641045331955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682648068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09674691408872604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682663526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363379865884781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682678663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508956015110016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682693727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940829128026962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682708784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039276272058487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682723948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445026308298111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682738901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106603667140007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682754113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364229768514633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682754114, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682754114, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682839998, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787365317344666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682839999, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682839999, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682854859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953126847743988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682870064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049618124961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682885024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425511002540588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682900158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881608724594116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682915053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022578775882721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682930226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09722264856100082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682945116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267947614192963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682959688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169193685054779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682974456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102775692939758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682989368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09478280693292618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683004261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040741428732872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683019414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211119592189789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683034769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159993171691895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683049742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758559197187424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683049743, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683049743, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683134792, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788498401641846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683134792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683134792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683149928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003885343670845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683164807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09708064794540405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683180025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577516257762909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683195660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239761114120483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683211074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771836131811142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683226344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09625890105962753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683241603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060340479016304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683256731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011858880519867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683272017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126181691884995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683287025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453146696090698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683301579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647908598184586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683317371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013389378786087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683332553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991099625825882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683347877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420777440071106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683347878, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683347878, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683431461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791565656661987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683431462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683431462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683446884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200584471225739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683462455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079196631908417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683477972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650143772363663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683493375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12041407823562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683508638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162724494934082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683524050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085723549127579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683539503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795628815889359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683554664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381060630083084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683569836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569417476654053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683585030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699986666440964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683600387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714977979660034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683615420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610151290893555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683630769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682041943073273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683645603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586028337478638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683645604, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683645605, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683729364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779501736164093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683729365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683729365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683744722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149322986602783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683759848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399642586708069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683775169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249096900224686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683790718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604654043912888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683806269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841739177703857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683821444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538666695356369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683836823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470211505889893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683852052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378294974565506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683867687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11808822304010391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683883190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548596829175949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683898702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644231528043747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784683914078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214455962181091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784683928807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788727551698685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784683944264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160818487405777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784683944264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784683944265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684027838, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797027826309204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684027839, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684027839, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684042757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115838050842285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684058249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111550472676754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684073711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11535286158323288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684089184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152465850114822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684104772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993957668542862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684120170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810161381959915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684135484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985700935125351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684150681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824607312679291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684166092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037233680486679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684181179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052270233631134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684196470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526396870613098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684211735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025506258010864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684226886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784093827009201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684242386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263345181941986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684242386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684242387, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684324194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799981236457825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684324195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684324195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684339639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193428188562393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684371101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648433119058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684386456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120548844337463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684409605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11658936738967896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684424958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262182891368866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684454239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11686275899410248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684469907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934070706367493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684486084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826950520277023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684501824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807077586650848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684521995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080017164349556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684537779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11824564635753632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684559916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082987487316132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684575545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429419577121735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684593974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158625781536102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684593976, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684593977, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684675680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800657749176025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684675681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684675681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684692575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547368973493576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684707986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117233335971832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684723151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889274418354034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684738666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567892342805862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684753374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526358336210251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684768307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117437690496445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684783871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615232586860657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684798882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948555171489716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684813973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300502717494965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684828768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09311604499816895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684843788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693289130926132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684858710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071970522403717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684873205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09796854108572006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684887851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742899775505066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684887852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684887852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684968671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799180150032043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684968672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684968673, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784684983554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097391158342361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684998548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09184364229440689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685013590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0963878482580185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685028616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924379527568817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685043670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813269764184952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685058629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205624788999557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685073329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09495174139738083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685088218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063651368021965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685103063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637501627206802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685117845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016860604286194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685132621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832217544317245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685147913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09884101152420044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685162882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11130787432193756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685177615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853550046682358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685177615, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685177616, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685259564, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802433371543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685259565, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685259565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685274706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060217097401619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685289643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09556010365486145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685304759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334471613168716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685320040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597670286893845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685335195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021464467048645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685350114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148891597986221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685365152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694924741983414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685380017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09534692019224167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685395004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481105536222458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685410117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404258221387863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685424787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294898599386215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685440402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10188038647174835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685455326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309437662363052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685470399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140959918498993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685470400, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685470401, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685553822, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802426815032959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685553823, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685553824, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685569225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023621469736099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685584363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09900522232055664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685599615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133899748325348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685614483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265591740608215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685629876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09884487837553024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685645008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007550060749054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685659889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689755737781525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685674822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029678046703339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685689984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035233810544014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685704975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838811844587326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685719997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043303832411766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685734992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755548626184464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685750119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206685960292816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685764830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09600643813610077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685764831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685764831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685849072, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803417444229126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685849073, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685849073, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685864110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09409871697425842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685878750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994352474808693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685893476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09688679873943329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685908036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022024810314178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685922836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425805300474167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685938144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09541871398687363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685952993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254562646150589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685968006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09814518690109253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685982835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815218299627304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685997746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104596808552742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686012539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09333734959363937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686027591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676295310258865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686042977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497158765792847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686058424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976212471723557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686058425, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686058426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686143501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804283499717712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686143502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686143502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686158463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562677681446075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686173376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108567029237747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686188428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541152954101562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686203774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10290229320526123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686218217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0964650809764862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686232943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445129126310349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686248197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566338360309601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686263195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410368233919144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686278264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832635521888733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686293859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09015043824911118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686309048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200516134500504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686324146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538216680288315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686339110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431114584207535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686354187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895597189664841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686354188, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686354188, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686437895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7806271910667419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686437896, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686437896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686452859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003699079155922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686467594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487233847379684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686482612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817977994680405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686497777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943222999572754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686513067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040724277496338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686527934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09505974501371384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686542927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461347550153732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686558225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294365137815475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686573206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100703164935112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686588649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014919728040695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686603517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09751194715499878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686618889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978830814361572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686633942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09958916902542114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686648947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534138232469559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686648948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686648948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686732065, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805741429328918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686732066, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686732066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686747311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016494557261467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686762607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09549698233604431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686778010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09779186546802521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686793206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794572532176971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686808782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285071283578873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686824282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08722840249538422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686839882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583741962909698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686855432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09293760359287262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686870877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048056036233902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686886314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09583890438079834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686901647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526937454938889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686916552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244292765855789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686931835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188504099845886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686947037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0978153645992279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686947037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784686947038, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784687032225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7809751033782959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784687032227, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784687032227, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784687046966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09515947848558426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687062436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551017731428146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687077362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120581835508347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687092394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294757038354874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687107686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335270315408707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687122985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568339377641678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687138366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824423283338547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687153318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589077860116959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687168288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887610912322998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687183366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09861230850219727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687198419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096364259719849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687213575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962110385298729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed445449777.log b/recommendation_v4/rcp_logs/gbs_8192/seed445449777.log new file mode 100644 index 000000000..477d68d9d --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed445449777.log @@ -0,0 +1,1830 @@ +:::MLLOG {"namespace": "", "time_ms": 1784658272109, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784658272109, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784658293692, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "seed", "value": 445449777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784658293694, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784658293695, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784658293695, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784659177290, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784659177290, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784659177644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784659294532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385040134191513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784659306651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391722410917282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659319030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860109448432922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659331918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903427124023438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659344634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902097940444946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659357454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387871503829956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659370334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904257118701935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659383353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868319988250732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659396287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895158469676971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659409134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388815939426422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659422473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385752409696579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659435659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860474526882172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659448790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898178935050964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659461871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872160017490387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659461872, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659461872, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659546525, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003085136413574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659546525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659546526, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659559475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879281282424927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659572738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875100016593933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659586422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860586285591125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659599425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388510912656784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659612763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879604637622833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659626666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138803631067276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659639849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870464265346527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659653252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386973261833191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659666949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849572837352753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659680224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871446251869202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659694102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865667581558228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659707900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858886063098907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659721511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855072855949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659735542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854500651359558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659735543, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659735543, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659818968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500533938407898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659818969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659818970, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659832266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383405327796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659845711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843666017055511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659859139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789646327495575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659873054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788044452667236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659886640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816708326339722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659900326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383134126663208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659914015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788366317749023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659928054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761326670646667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659941614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736452162265778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659954971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786755502223969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659968529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747118413448334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659981836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795284926891327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659996068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806922733783722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660009881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790225982666016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660009882, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660009882, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660092682, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008573532104492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660092683, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660092683, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660106127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716761767864227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660119927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769954442977905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660133599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731221854686737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660148014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137658953666687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660161721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376846581697464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660175598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376277506351471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660189355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13694307208061218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660203024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775019347667694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660217084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13709893822669983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660230935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385861486196518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660244765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653744757175446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660258493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805636763572693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660272311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581249117851257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660286500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13685853779315948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660286500, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660286501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660371704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013286471366882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660371705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660371705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660385257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366022229194641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660398924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577473163604736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660412331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375255584716797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660426000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349341720342636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660439895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13689658045768738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660453532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338019222021103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660467688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563168048858643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660481425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558539748191833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660495050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377965211868286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660508494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531076908111572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660522013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14599379897117615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660535918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563737273216248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660549517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421490788459778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660563337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789433240890503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660563338, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660563338, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660648086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019841194152832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660648087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660648087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660661727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355731338262558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660675423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13943177461624146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660689055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597846031188965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660702764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332540512084961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660716646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135092094540596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660730731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907773792743683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660744531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362634181976318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660758413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312415778636932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660772171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330930471420288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660785797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175931572914124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660799667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118524849414825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660813266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521181046962738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660827047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321125030517578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660841154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896715641021729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660841154, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660841155, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660926618, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502794086933136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660926619, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660926619, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660940549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382405042648315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660954419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333058625459671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660968361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596372306346893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660982063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134818434715271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660995760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783268630504608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661009534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278054565191269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661023005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542217016220093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661036647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920928001403809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661050504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354243338108063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661064673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262854516506195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661078496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366079330444336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661092700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345110982656479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661107373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554905354976654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661121465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356848061084747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661121465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661121466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661205393, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503851592540741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661205394, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661205394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661219570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256527483463287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661233594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361167907714844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661247693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356119066476822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661262094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538873195648193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661276292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396019607782364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661290273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324999809265137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661305216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533097505569458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661319563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347502052783966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661333747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277536630630493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661348036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199873268604279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661362008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566499948501587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661376420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374212384223938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661390415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331852853298187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661404289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060279190540314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661404290, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661404291, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661490405, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5050032734870911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661490406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661490406, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661505321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791999220848083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661519844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350167691707611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661534341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540539145469666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661548857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124504685401917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661563164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375398516654968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661577556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399969041347504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661592070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13705812394618988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661606577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387799263000488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661620965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13301748037338257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661635043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12965385615825653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661648876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771254777908325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661662928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298986852169037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661677130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297127902507782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661690926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12475553900003433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661690926, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661690927, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661776481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5062720775604248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661776482, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661776482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661790587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114920258522034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661804591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605161011219025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661819026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347499072551727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661833415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031218945980072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661847419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283774971961975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661861765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12669014930725098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661876196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289454698562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661890195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236573338508606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661904257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562940061092377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661918430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12484090030193329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661932370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287190467119217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661946391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419446349143982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661960578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107553124427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661975040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926720082759857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661975041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784661975041, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662059671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5078045725822449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662059672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662059672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662073933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304478943347931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662087972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12899984419345856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662101633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331644505262375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662116176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384305894374847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662130476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952300906181335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662144262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259061098098755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662158101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010390102863312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662172063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218615472316742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662185627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611231744289398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662199673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703440129756927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662213488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12356889992952347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662227939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12334201484918594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662242152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568338215351105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12908585369586945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662256252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662256252, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662341223, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5095950365066528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662341224, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662341224, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662355012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637247681617737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662369253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665024936199188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662382966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567212224006653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662396600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399870067834854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662410366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675029575824738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662424249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833028078079224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662438048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740832567214966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662451826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12614795565605164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662465557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223341643810272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662479709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14121288061141968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662493932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714490294456482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662508062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856099009513855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662522184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304686546325684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662535747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139653205871582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662535747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662535748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662621101, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5114534497261047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662621102, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662621102, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662635862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14146853983402252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662650277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14241482317447662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662664342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555076718330383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662678570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955754041671753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662693003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14497242867946625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662706992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14090517163276672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662721333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258312612771988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662735723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756626844406128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662750185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183952867984772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662764702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614732027053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662779158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774849474430084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662793382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330164521932602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662808188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433340191841125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662822810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363224983215332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662822811, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662822811, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662906114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5131489038467407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662906116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662906116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662920218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867628574371338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662934698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806012272834778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662949322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170631229877472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662963799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282501697540283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662978203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340297907590866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662992887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13993069529533386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663007424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328442394733429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663022105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410883218050003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663036165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14092837274074554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663050826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517029583454132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663065062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278383731842041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663079276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13003265857696533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663093431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340280920267105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663107753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12783387303352356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663107754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663107754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663190436, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5153077840805054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663190437, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663190437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663204637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201507925987244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663218802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681241869926453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663233063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363030552864075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663247340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318785548210144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663261630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13406118750572205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663275737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353371620178223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663290321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370196521282196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663304769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253095746040344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663319027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13996823132038116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663333940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196982443332672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663348143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289006769657135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663362115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311865597963333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663376232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325758993625641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663390437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13598774373531342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663390437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663390438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663473385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182751417160034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663473386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663473386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663487579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336842179298401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663501681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12280520796775818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663515856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049151003360748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663530083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303102970123291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663544243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534431159496307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663558886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351184844970703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663573090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553860783576965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663587733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12404460459947586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663602032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110758364200592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663615993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311349719762802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663630776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012734055519104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663645055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330265879631042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663659617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308409422636032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663673807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301446557044983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663673808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663673808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663756762, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5219566226005554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663756763, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663756764, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663771178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378519982099533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663785619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239964842796326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663799754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370469212532043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663813933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044136762619019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663828063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718911468982697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663842109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188238441944122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663856727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355810910463333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663871329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295339047908783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663885734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145066797733307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663900357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916756212711334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663914821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393810391426086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663928662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771072030067444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663943313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13223223388195038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663957560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353055715560913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663957560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784663957561, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664040919, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5269417762756348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664040919, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664040920, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664055124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109833002090454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664069356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501331210136414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664083444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783860206604004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664097703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325060904026031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664111801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740154564380646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664125848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12934662401676178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664139886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259523332118988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664153873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12417270243167877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664168137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13624532520771027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664182542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373194456100464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664196935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469500839710236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664211858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271541714668274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664226217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090214133262634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664240110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12371795624494553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664240111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664240111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664324554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5336902141571045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664324555, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664324556, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664339048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250072300434113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664353556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261140882968903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664368045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338394433259964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664382527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380421370267868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664396823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13632096350193024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664411310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131511390209198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664425553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12731605768203735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664439881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497331738471985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664454025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12488988786935806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664468408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307215690612793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664482774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325303465127945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664497235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447019457817078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664511739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14028140902519226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664525956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363411396741867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664525957, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664525957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664611431, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5437871813774109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664611432, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664611432, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664626159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679808378219604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664640468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159193098545074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664654507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270489871501923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664668787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391608864068985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664683610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300201117992401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664698529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13856743276119232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664712862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516876101493835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664727449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13743507862091064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664741792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522303104400635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664756426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061872124671936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664770966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14082814753055573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664785512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13488323986530304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664800068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922334671020508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664814734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133436381816864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664814735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664814736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664899963, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5562683939933777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664899964, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664899965, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664914891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14219091832637787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664929366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293559849262238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664944462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366325199604034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664959108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13086801767349243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664974185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540861010551453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664989066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373404741287231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665003718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432857394218445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665018391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577519357204437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665032936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131750226020813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665047865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297758996486664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665062599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289733231067657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665077620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719181716442108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665092220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620123267173767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665106963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13765430450439453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665106964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665106964, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665190783, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5718265175819397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665190784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665190784, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665205417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194261491298676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665220334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541091978549957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665234977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346476525068283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665250205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216978311538696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665264766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374480426311493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665279358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002115488052368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665294189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387461006641388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665308423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12462623417377472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665322894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315898597240448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665337157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13197967410087585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665351506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142399489879608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665365639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209949433803558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665379787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12913814187049866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665394121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13135449588298798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665394121, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665394122, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665475606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5872239470481873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665475607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665475607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665489710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231083795428276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665504179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747760117053986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665518508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000479340553284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665532928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341046541929245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665547704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237379491329193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665562114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12269998341798782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665576848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128764927387238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665591462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13255879282951355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665606092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778647780418396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665620246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12908224761486053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665635159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419172167778015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665649549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.118312306702137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665663800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12278629094362259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665678238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12610751390457153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665678238, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665678239, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665759727, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6029505133628845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665759728, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665759728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665773991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222175881266594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665788046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12834540009498596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665802254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12943239510059357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665816332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285161554813385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665830409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226353645324707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665844776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356327086687088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665858853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023331761360168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665872927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12240096926689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665887290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800364196300507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665902002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12878865003585815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665916368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11954698711633682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665931324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233055174350739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665946107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12799927592277527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665960750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12910166382789612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665960751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784665960751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666043845, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6165855526924133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666043846, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666043846, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666057999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12899810075759888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666073048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11958838999271393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666087610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12287405133247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666102371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12231510877609253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666117051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1242271214723587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666131551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267382651567459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666146181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11767017096281052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666160943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12362334132194519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666175827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126207172870636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666190506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12434735149145126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666204990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645860970020294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666219581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12434536218643188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666233920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109306663274765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666248572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12544524669647217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666248573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666248573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666331132, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6281856894493103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666331133, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666331133, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666345942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175813302397728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666360790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597001552581787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666375471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13036756217479706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666390467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248865514993668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666405159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1241280660033226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666419750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752587556838989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666433927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12046768516302109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666449048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169107794761658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666463695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163486331701279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666478546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401055008172989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666493651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12078645080327988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666508393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187737226486206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666523209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1241844892501831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666538291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763212829828262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666538291, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666538292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666621592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6436138153076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666621593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666621594, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666636479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11968015879392624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666651263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11195351928472519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666665923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11045076698064804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666680592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325675249099731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666695117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887802392244339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666709828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303686887025833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666724644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633007973432541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666739817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12056350708007812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666754714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309607535600662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666770117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11949978768825531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666785310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125161349773407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666800433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341500282287598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666815177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968302190303802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666830004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11636824160814285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666830005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666830005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666912291, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6684707999229431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666912292, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666912292, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666926651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11990544945001602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666941268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004435807466507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666956033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236724704504013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666970686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591008514165878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666985290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11441670358181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666999766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11722967773675919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667014552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1193791851401329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667029276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11855858564376831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667043841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10891126096248627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667058401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11707554012537003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667072946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11651825904846191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667087064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674211174249649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667101401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279110610485077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667115980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12598268687725067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667115980, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667115981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667197336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6964637041091919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667197337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667197337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667212066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12429320812225342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667226709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657688766717911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667241255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682773381471634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667256232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227530911564827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667271005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097065880894661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667285798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137907952070236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667299937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310146003961563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667314821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829786211252213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667329864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126122623682022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667345342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614967882633209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667360696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12282561510801315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667375718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565254628658295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667391044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12656131386756897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667406051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175753176212311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667406052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667406052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667488906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7423039078712463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667488907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667488907, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667504084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12671229243278503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667518845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370538175106049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667533487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704907566308975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667548488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263636499643326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667563259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872576922178268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667578281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188865453004837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667593440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214253306388855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667608570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022525817155838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667623386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11640334874391556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667638961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370821297168732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667653927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649208515882492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667668699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11602320522069931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667683403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12517361342906952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667698478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489777266979218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667698479, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667698479, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667783587, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7547053694725037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667783588, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667783588, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667798334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158678472042084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667813266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049844115972519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667828159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373749375343323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667843220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460613459348679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667857997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830501466989517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667872943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510661244392395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667887863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11830668896436691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667902444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134132742881775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667917313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956967622041702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667932136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803871601819992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667947321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12329375743865967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667962483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657834261655807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667977370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210867315530777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667992619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12068542838096619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667992621, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784667992621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668076814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7593855857849121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668076815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668076815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668091805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11957179754972458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668106505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11944931000471115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668121342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537792533636093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668136619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11585044860839844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668151496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338321328163147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668166313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036469042301178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668181105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889645457267761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668195845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11512161791324615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668210493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037197932600975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668225119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322792828083038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668239464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662948340177536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668254137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211103945970535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668268838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151934415102005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668283447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11480628699064255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668283487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668283487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668366774, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7612780332565308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668366775, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668366775, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668381249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042030081152916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668396116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709629207849503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668411105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499354988336563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668425854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082373559474945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668440644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753297060728073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668455860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463724821805954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668470590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11304540187120438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668485193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314819008111954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668499907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601400375366211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668514463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061184853315353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668528854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223202407360077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668543432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11329970508813858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668557855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728388279676437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668572216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652810335159302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668572217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668572217, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668654027, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634466290473938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668654028, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668654028, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668668496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063060387969017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668683156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11356443166732788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668697742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959916040301323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668712336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372047126293182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668726857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300912499427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668741189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489965230226517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668755683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08879072964191437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668770121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907050967216492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668784659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166241228580475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668799384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11808981746435165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668814200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752058029174805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668828574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072484612464905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668843527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056548565626144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668858362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740729421377182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668858362, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668858363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668940878, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640739679336548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668940878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668940879, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668955452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603651195764542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668970199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248176544904709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668984423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324057936668396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668999792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543646663427353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669014350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887168556451797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669029188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208022594451904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669043979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591927170753479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669058753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864003002643585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669073554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09735401719808578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669088548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939139127731323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669103096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501661151647568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669117778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715468972921371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669132477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835100710391998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669147308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274756491184235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669147309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669147309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669230655, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652957439422607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669230656, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669230656, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669245250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158257722854614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669259483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993644967675209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669274174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296499729156494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669288995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595682263374329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669303970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075560003519058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669319081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388204991817474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669333914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005367636680603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669348859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605353862047195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669363786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718484222888947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669378511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642769932746887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669393435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253490298986435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669407630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09423387795686722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669422749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836533665657043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669437514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072072833776474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669437515, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669437515, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669520563, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7670378088951111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669520564, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669520564, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669534845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09248369932174683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669549464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529439896345139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669563890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11054576188325882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669578534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811338037252426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669592982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023346334695816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669607605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097199469804764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669622189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711474925279617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669636688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115062236785889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669651253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540974140167236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669665678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368084907531738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669680349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809589177370071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669694623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546012222766876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669709032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09706605225801468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669723729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598652809858322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669723729, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669723730, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669806308, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678406238555908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669806309, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669806309, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669821192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421591252088547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669836216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477129369974136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669850957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556849092245102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669865944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440590232610703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669880877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09829280525445938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669895768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10007564723491669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669911015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645541548728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669925471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09659729897975922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669940402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811535269021988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669955141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868263989686966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669969787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699599027633667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669984667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491916328668594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669999348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626185476779938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670013932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506538093090057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670013933, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670013933, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670095528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688217759132385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670095529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670095529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670109889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056958347558975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670124690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556973308324814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670139080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073557510972023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670153801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152551531791687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670168409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09061499685049057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670183136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09802178293466568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670197571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440539568662643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670212310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742942243814468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670226653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768548399209976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670241205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115746408700943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670256309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328820139169693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670271355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057849436998367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670286293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962004959583282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670300860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145118623971939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670300861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670300861, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385081, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691139578819275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385082, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670399772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810726881027222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670414575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10179270058870316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670429132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044004559516907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670444004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09199096262454987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670458338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475147306919098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670472665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637982189655304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670488029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076415330171585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670502961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153299361467361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670518077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12168435752391815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670533055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472878813743591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670547942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139868199825287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670563128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12310938537120819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670578073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968694299459457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670592960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144192516803741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670592961, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670592961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670674026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7694743871688843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670674027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670674027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670688919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461154580116272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670703883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11844455450773239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670718687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921897739171982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670733791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620278865098953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670748702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09662234038114548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670763604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131836548447609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670778942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618959367275238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670794082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602917522192001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670809360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11693654209375381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670824221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949507564306259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670839443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12520316243171692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670854614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756687819957733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670869639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167549416422844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670884599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780352354049683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670884599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670884600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670967915, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7695014476776123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670967916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670967916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670983065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440563946962357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670998165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496679484844208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671013271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147721856832504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671028603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776152998209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671043864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142815351486206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671059079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11852502077817917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671074233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12731370329856873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671089120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109313160181046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671104207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163223385810852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671119708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11060924828052521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671134787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829655081033707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671150083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794391483068466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671165229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11530078947544098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671180608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146334633231163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671180609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671180609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671263253, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705512046813965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671263254, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671263254, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671278647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870265752077103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671293997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753200203180313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671308978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088855043053627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671324603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817350447177887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671340010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105554923415184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671355334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940834879875183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671370580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137402206659317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671385783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081015020608902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671400535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614008456468582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671415134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015848278999329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671430157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425149649381638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671444929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686174243688583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671459690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431080311536789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671474297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053648054599762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671474298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671474299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671555517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714986801147461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671555518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671555518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671570019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965543240308762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671584690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135402530431747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671599427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09624281525611877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671614149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108899787068367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671628952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1154002919793129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671643671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438907146453857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671658434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042509377002716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671673232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457712411880493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671687917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020629033446312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671702874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120290108025074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671717803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682896316051483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671732647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139837861061096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671747836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023221373558044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671762598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12038003653287888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671762599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671762600, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671843581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718601822853088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671843582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671843582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671858852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046811938285828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671873715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09648442268371582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671888378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467661172151566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671903082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286858677864075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671917587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853414237499237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671932348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454616695642471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671947235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606733709573746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671962030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239464581012726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671977121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467716306447983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671992075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801473259925842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672006905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542131215333939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672021615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506766080856323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672036121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694839060306549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672050893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216545313596725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672050893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672050893, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672132830, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722415328025818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672132832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672132832, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672147713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983208775520325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672162774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0942969098687172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672177423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959373489022255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672192405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09634906053543091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672207114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09659279882907867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672222093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0943305641412735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672236847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039058327674866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672251672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09708424657583237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672266960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355392098426819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672282136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830309987068176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672297668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488438606262207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672312921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255158692598343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672328122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11410194635391235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672343679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099975556135178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672343680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672343680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672425169, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725469470024109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672425170, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672425170, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672440323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853262454271317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672455337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723048448562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672470590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093062162399292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672485236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144847631454468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672499948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011178255081177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672514877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077674925327301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672529709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075367778539658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672544571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064181923866272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672559752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243790596723557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672574544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129999369382858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672589423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001054406166077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672604093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247248411178589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672619037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931456089019775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672633745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418963223695755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672633747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672633747, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672715442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726956605911255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672715444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672715444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672730247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019807308912277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672745051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644898563623428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672759820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09108684957027435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672774388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261962562799454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672789322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661717504262924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672803917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802407562732697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672818642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020546406507492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672833598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029590368270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672848645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975637495517731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672863722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041017696261406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672878865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294979810714722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672893691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811974108219147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672908978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513974726200104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672924126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905253142118454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672924127, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672924127, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673006418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7729923725128174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673006419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673006419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673021419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360193252563477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673036278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027137041091919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673051012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422030091285706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673065387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288931429386139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673080530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720425099134445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673095089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065748542547226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673109906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09831822663545609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673125009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228090733289719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673139955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204880475997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673154706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503429919481277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673169294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368529707193375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673184062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243048518896103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673199135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407949239015579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673214065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649300366640091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673214066, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673214066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673296957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732399106025696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673296959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673296959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673311964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851900279521942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673327111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147605091333389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673342128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409355908632278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673357192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110916197299957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673372308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461244732141495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673387039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255928874015808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673401648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104639828205109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673416119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001124233007431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673430986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312329769134521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673446432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655687004327774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673461459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032642051577568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673476665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473445057868958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673492116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051735207438469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673507390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09851009398698807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673507390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673507391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673590461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773639976978302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673590462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673590462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673605676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224621742963791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673620628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954150766134262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673635475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643717646598816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673649877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064958781003952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673665034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787218809127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673679685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937006771564484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673694496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717037320137024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673709432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546512693166733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673724276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394838243722916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673739011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022850334644318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673753748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410064458847046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673768517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10271741449832916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673783202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050671711564064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673797824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10719837248325348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673797824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673797825, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673881151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735855579376221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673881152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673881152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673895802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202959179878235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673910490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041125863790512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673925526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857228189706802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673940283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050824448466301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673955003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09375472366809845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673970187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273176968097687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673984945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117126256227493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673999786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779134184122086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674014761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033160388469696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674029987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398197174072266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674044917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901445895433426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674060014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414525657892227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674075200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616274178028107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674090479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294228047132492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674090480, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674090481, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674172362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773617684841156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674172363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674172363, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674187717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11273746192455292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674202814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085951030254364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674217795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924539715051651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674232805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682352632284164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674247507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121100395917892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674262128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721360146999359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674277571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11847499758005142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674292638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908778756856918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674307857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632179468870163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674323250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299729138612747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674338450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734564810991287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674353579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058859527111053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674368778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474892497062683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674383752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010472476482391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674383754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674383754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674465512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732471823692322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674465513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674465513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674480709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11348290741443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674496010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937212407588959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674511055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017338186502457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674526210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686174035072327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674541238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041201576590538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674556438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361793428659439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674571514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733374953269958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674586747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473297327756882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260194331407547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674617576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12019200623035431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674633077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057654544711113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674648433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758798360824585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674663589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736441612243652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674679079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11927633732557297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674679079, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674679080, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674762041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736504077911377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674762042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674762042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674777272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192304790019989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674792600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792165249586105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674807701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194903627038002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674822433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136130839586258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674837348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736623406410217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674852887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526527255773544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674867866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154192686080933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674883293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698611825704575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674898697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277275532484055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674913997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606300830841064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674929201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257407814264297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674944573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583767294883728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674959914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678359657526016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674975581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515813529491425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674975581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784674975582, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675059730, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740634083747864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675059731, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675059731, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675074961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097182035446167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675090512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1204419806599617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675105958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975834727287292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675121768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349661648273468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675137189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624829471111298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675152458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927244275808334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675167785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818178951740265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675183228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977223515510559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675198899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297088116407394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675214219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11182568967342377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675229586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103486642241478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675245021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090102791786194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675260491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11104629188776016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675275923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883808881044388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675275924, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675275924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675361262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744095325469971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675361263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675361263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675376373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791641265153885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675391683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013548076152802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675406691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073222756385803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675421434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278283804655075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675436241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585616528987885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675451465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151833087205887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675466065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789678245782852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675480710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725130885839462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675495287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857775270938873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675509898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173670947551727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675524433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521489381790161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675538863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048695519566536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675553597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977935045957565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675568508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537409573793411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675568509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675568510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675652270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746921181678772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675652270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675652271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675666904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451999843120575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675681607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513345152139664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675696508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005859375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675711456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09214384108781815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675726230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428915709257126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675741093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381657630205154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675755935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235056281089783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675770976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278576403856277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675785802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673633217811584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675800847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341820120811462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675815753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935336351394653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675830769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064570397138596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675845894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074945405125618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675860845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069459468126297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675860846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675860846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675943955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746965289115906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675943956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675943956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675958786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978227853775024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675973692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315960645675659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675988300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056045442819595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676002993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09896985441446304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676018441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589457839727402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676033640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002589017152786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676048653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343604534864426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676064018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795431584119797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676079375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853412747383118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676094260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830919444561005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676109422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023123562335968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676124568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994846373796463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676139460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669802129268646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676154333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08741606026887894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676154334, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676154335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676238227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749086022377014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676238228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676238229, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676253300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0953153744339943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676268128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109287366271019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676283098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712539404630661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676297925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112056136131287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676312959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10215352475643158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676327923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09023141860961914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676342821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361332446336746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676357603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09746335446834564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676372507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415627062320709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676387796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638593137264252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676403085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023956537246704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676418343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811512917280197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676433259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638315975666046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676448516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349105298519135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676448517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676448518, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676530496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749032378196716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676530496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676530497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676545543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962458997964859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676560754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515347123146057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676575758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09469259530305862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676590816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254253447055817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676605920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09776675701141357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676620504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037637501955032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676635061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971267521381378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676650598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162622481584549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676665748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901549458503723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676681028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980881005525589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676696711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647410899400711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676712426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422652214765549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676727827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919458419084549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676743313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877827554941177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676743314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676743314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676825376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775143563747406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676825377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676825378, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676840612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621056705713272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676856064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444435477256775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676871426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352024435997009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676886802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10025523602962494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676902090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717789083719254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676917595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09599078446626663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676933624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522492974996567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676948942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301094502210617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676964153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040294915437698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676979571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09890804439783096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676994685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766446799039841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677009725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201621800661087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677024972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09080013632774353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677040298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708419233560562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677040298, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677040299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677124150, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775304913520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677124151, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677124152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677139376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954556077718735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677154833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10124719887971878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677170194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09938570857048035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677185881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976489633321762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677201652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039476305246353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677217564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231221467256546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677233156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10276664793491364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677248755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052609235048294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677264480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147903114557266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677279655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177193373441696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677295095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104475915431976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677310054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817451030015945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677325038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559087246656418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677340346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621187835931778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677340347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677340347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677423718, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753314971923828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677423719, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677423719, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677438916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065097525715828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677454009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819330275058746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677468940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680083930492401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677484149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659655928611755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677499314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102410688996315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677514100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710478574037552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677529049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09765323996543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677544242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09798505902290344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677559272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557659715414047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677574325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427972674369812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677589511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264332592487335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677604792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991820693016052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677620044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672436654567719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677635343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976138919591904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677635344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677635344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677716917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755942940711975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677716918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677716918, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677731948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307260602712631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677747104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531698167324066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677762319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09444250911474228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677777822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853300452232361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677792774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186491161584854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677808427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024400800466537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677823742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723703354597092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677839214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975366085767746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677854648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889792442321777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677870222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912896156311035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677885642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250487923622131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677900605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898407012224197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677916206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199508607387543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677931319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050741374492645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677931319, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677931320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678015042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757527232170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678015043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678015043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678029884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561197996139526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678045459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495955497026443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678060483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634572058916092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678075538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103934049606323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678091066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648325085639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678106653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857702046632767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678121943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758531838655472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678137242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09703832119703293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678152560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684499889612198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678167714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363306850194931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678182941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847792774438858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678197815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364038497209549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678213254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667382925748825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678228402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09583313763141632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678228449, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678228450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678310928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757769227027893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678310929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678310929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678326055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963405877351761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678341153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568317025899887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678356309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026768758893013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678371438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10423463582992554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678386366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845762491226196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678401120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12268467992544174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678415975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805606096982956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678431006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912183880805969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678446406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126422882080078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678461562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117323487997055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678476976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148165613412857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678492337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09866204112768173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678507876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655929148197174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678523352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412930697202682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678523397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678523397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678605595, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761412262916565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678605597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678605597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678620996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09763433039188385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678636426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09793169796466827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678651745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125013455748558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678667141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754307895898819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678682300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391586065292358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678697211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075018346309662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678712858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072455644607544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678728169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043075263500214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678743411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489626973867416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678759042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830383002758026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678774522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078112572431564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678789813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024985462427139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678805282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307384073734283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678820859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103109270334244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678820859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678820860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678905278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762284278869629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678905279, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678905279, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678920850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720402747392654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678936345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021386831998825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678951959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427932977676392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678967300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022694408893585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678982627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821555554866791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678997950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788659751415253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679013131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014896109700203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679028088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270152032375336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679043342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12082923948764801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679058595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092170849442482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679073659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843342542648315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679089150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411018133163452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679104512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388119518756866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679119808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880380868911743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679119809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679119809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679203215, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763116359710693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679203216, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679203216, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679218682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401378363370895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679234254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11880327016115189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679249610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382234841585159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679265151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11236588656902313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679280664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115028709173203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679295913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11911268532276154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679311354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187660694122314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679326309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883311182260513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679342030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048910766839981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679357589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989348292350769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679373081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035005003213882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679388979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10917095839977264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679404640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041174903512001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679419824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034511774778366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679419824, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679419824, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679503458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765874862670898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679503458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679503458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679518845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279422789812088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679534523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12851126492023468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679550111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785603523254395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679565464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212355643510818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679580776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270568519830704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679596356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163929849863052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679611910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419732123613358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679627411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242497712373734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679642773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601702332496643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679658215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812132060527802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679673930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960118055343628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679689429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293582618236542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679705323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252341419458389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679721087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593203246593475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679721088, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679721088, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679805805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768086791038513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679805806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679805806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679821109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09503095597028732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679836716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143868416547775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679852493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1220734715461731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679868349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689578205347061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679884212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924811661243439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679899976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748913884162903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679915602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170747876167297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679930850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309115588665009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679946180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334143787622452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679961221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11789017915725708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679976465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071835458278656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679991924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154791712760925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680007453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09195210039615631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680022579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329702496528625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680022580, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680022580, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680106513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772626876831055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680106514, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680106514, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680121847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152880102396011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680137108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09651099145412445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680152184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447761952877045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680167279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771961510181427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680182341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951607674360275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680197745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393239557743073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680213117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801416635513306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680228647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332305520772934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680243975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924273192882538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680259478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526634216308594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680274839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677182883024216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680290186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09682827442884445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680305331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239659249782562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680320699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042516142129898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680320699, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680320700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680405288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774080038070679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680405288, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680405288, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680420840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343273729085922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680436464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255222022533417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680451903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411445796489716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680467434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670411586761475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680482917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204163938760757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680498498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992836132645607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680514084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929471254348755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680529274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531695932149887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680544573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690239071846008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680559884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564370453357697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680575341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427649319171906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680590251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790415853261948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680605033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589414089918137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680620394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129705086350441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680620433, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680620433, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680704708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776393890380859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680704709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680704709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680719690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803785175085068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680734819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147508770227432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680750257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699281841516495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680765755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739461332559586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680780824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000911146402359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680796119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021203100681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680811622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09574586153030396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680826905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733816027641296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680842068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935653209686279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680857483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024320125579834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680872991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142025351524353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680887995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294774174690247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680902881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936762601137161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680918128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037883311510086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680918172, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680918173, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681002042, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779293060302734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681002043, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681002043, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681017169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09783514589071274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681032193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709939152002335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681047131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09923037141561508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681062108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263077914714813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681076903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09431903064250946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681091909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648668557405472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681107052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346520692110062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681122001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339540243148804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681137283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496792942285538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681152347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204688251018524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681167774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09604917466640472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681182971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111388199031353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681198156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220068484544754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681213070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160427540540695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681213118, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681213118, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681296458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7782747745513916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681296459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681296459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681311437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09766505658626556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681326652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694130510091782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681341918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076536983251572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681356530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592563450336456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681371759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037766396999359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681386841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890448093414307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681401825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011798143386841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681417091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886784851551056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681432231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863588750362396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681447324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10879096388816833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681462184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990335255861282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681477283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09396615624427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681492135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.094964899122715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681506986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560403019189835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681506987, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681506987, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681591095, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788073420524597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681591096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681591097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681606126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316324979066849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681621074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352656781673431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681636219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107781738042831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681651264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051739752292633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681666145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057046800851822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681680929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10067886114120483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681695650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916092246770859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681710359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080328583717346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681725228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689371079206467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681739960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121692717075348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681754774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09728371351957321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681769902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156788468360901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681785128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934922635555267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681800174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894174128770828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681800175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681800175, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681884044, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792158722877502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681884045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681884045, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681899190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561653017997742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681914173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069450736045837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681929498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992402046918869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681944701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10044783353805542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681959877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488776117563248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681974926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989568904042244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681989842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002737358212471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682005014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610240697860718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682019660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545233637094498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682034181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09394663572311401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682049703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442201048135757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682064352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183911770582199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682079207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09321044385433197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682094201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003878191113472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682094202, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682094202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682178396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795929908752441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682178397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682178397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682193421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068933457136154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682208375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133204847574234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682223027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060106962919235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682237926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647847503423691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682252812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315808653831482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682267478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255585610866547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682282167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667102783918381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682296871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837776958942413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682311604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09762067347764969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682326255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09731391072273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682341108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086210608482361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682355561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824257344007492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682370524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210739076137543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682385083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027718037366867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682385084, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682385084, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682467364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800740599632263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682467365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682467365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682482139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015990749001503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682496793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785435885190964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682511603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11173856258392334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682526932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09399579465389252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682542103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329471528530121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682557235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761809349060059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682572499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576589941978455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682587434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107108950614929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682602649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560904443264008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682618115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969234257936478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682633399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09699670970439911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682648632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736818611621857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682663983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823979437351227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682678911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403389483690262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682678912, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682678912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682763181, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804328799247742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682763182, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682763182, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682778625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002206057310104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682793521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431156307458878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682807939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09204374253749847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682823472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003985464572906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682838476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084296777844429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682853526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071561276912689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682868826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09570522606372833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682883855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405900329351425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682898950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565535724163055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682913792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919320046901703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682928558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401365906000137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682943465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425493866205215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682958204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085358262062073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682973145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432752758264542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682973146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784682973147, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683058382, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805263996124268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683058383, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683058383, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683073154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092694029211998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683088361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045776978135109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683103193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457926988601685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683118305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843704640865326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683133087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10196291655302048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683148143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09715113788843155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683163127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260411351919174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683177898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221269518136978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683192898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1009652242064476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683207807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09509485960006714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683222801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484038293361664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683238113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214220523834229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683253452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161492228507996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683268456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11829638481140137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683268457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683268457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683353642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810918092727661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683353643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683353643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683368946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011789202690125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683383871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09712893515825272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683398923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057550236582756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683414175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123063862323761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683429420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755997687578201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683444732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09612757712602615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683459823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550062358379364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683474650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121006518602371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683489832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099315643310547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683504796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415304452180862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683519434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611975938081741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683535064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938828438520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683550152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993092805147171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683565396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497005075216293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683565397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683565397, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683649664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811068892478943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683649665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683649665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683665065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183054000139236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683680573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075765639543533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683695900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612843930721283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683711308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034787237644196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683726624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159222424030304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683742119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198116302490234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683757584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11811933666467667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683772711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363101750612259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683788131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510976612567902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683803708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692610591650009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683819371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712676495313644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683834793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615044832229614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683850488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715014487504959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683865582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581508070230484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683865582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683865583, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683947677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.78179532289505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683947678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683947678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683963191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202171444892883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683978404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343202203512192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683993840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241885483264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684009483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674439162015915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684025137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817829519510269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684040805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598837584257126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684056623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463307052850723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684072425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353691667318344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684088520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11831803619861603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684104358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456200689077377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684120017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116880401968956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684135537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120956763625145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684150679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783299803733826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684166277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544512212276459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684166277, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684166277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684248365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818892002105713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684248366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684248366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684263627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039455443620682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684279191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077302694320679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684294784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435970664024353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684310258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10087068378925323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684326444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957294166088104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684342053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812903195619583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684357556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09831515699625015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684372799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788055509328842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684388270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078393667936325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684403414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532651841640472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684418729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11434761434793472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684433994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971762984991074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684449058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690615117549896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684464513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128171756863594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684464513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684464514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684546423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782130777835846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684546427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684546427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684561702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119459718465805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684577134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697105526924133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684592603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076167970895767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684608079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11665022373199463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684623456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178120225667953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684639117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721538007259369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684655304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928639978170395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684670993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10822805017232895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684686980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069592610001564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684702675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864470899105072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684718515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181265339255333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684734227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239174008369446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684749582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335464030504227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684765179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128195375204086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684765180, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684765181, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684848251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7820969820022583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684848252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684848252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684863560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538104176521301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684879304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103857234120369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684894536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11958566308021545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684910085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610470920801163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684925047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548275709152222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684940227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143610835075378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684955929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064373031258583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684971146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962782055139542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684986600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319278180599213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685001735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09416196495294571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685017164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11671292036771774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685032395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007932648062706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685047318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857542812824249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685062265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781706124544144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685062266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685062267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685146098, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822530269622803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685146099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685146099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685161113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023843064904213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685176378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09373998641967773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685191620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09667449444532394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685206786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839026421308517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685221952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827963799238205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685236984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202222853899002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685251905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09572867304086685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685266972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642511397600174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685282073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572035610675812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685297029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11061465740203857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685312061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853202849626541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685327458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879030287265778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685342634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000529676675797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685357650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834957122802734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685357651, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685357652, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685441418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825146317481995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685441419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685441419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685456830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540813207626343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685472007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09572780877351761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685487403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388844460248947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685502866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606749564409256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685518437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235074162483215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685533802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141018569469452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685549247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672280937433243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685564328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0954878032207489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685579761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141490638256073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685595281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395628958940506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685610192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025371253490448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685626053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084431618452072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685640984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234826058149338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685656163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125065594911575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685656163, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685656164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685741406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828105688095093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685741408, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685741408, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685756756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069417953491211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685772145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09909168630838394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685787406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224876552820206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685802512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217224806547165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685817810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09870462119579315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685833064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109435766935349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685848148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961841568350792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685863233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903848171234131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685878543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319586843252182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685893716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827502608299255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685908733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448689013719559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685923838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755237191915512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685939159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178631544113159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685954140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09564744681119919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685954141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784685954141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686038246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7830361723899841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686038247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686038247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686053691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09401482343673706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686068828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889029711484909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686083994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711940586566925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686098932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965526312589645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686113935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032312661409378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686129487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09571346640586853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686144535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243865102529526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686159600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09794268012046814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686174527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09822866320610046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686189494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398364812135696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686204537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0928955003619194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686219877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720173269510269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686235263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043834462761879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686250489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818883031606674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686250490, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686250490, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686332950, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7834908962249756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686332950, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686332951, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686347924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419876128435135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686362987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144037753343582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686378015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051105260848999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686393560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270301252603531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686408377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0964488685131073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686423229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044936329126358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686438649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09517260640859604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686453871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367480456829071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686469111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09831000864505768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686484841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0897345021367073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686500484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996048361063004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686516119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454338043928146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686531452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034594178199768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686546620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945401340723038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686546621, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686546622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686626840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7835420370101929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686626840, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686626841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686641903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071173310279846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686656839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554233193397522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686671781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09723325818777084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686686977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09968752413988113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686702198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09987471997737885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686717160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09418851137161255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686732356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431268066167831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686747741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175753384828568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686762673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019813477993011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686778012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096058040857315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686792800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09763608127832413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686808173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986037015914917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686823501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09915543347597122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686838856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045573279261589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686838856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686838857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686920703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837992906570435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686920704, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686920704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784686935937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048116743564606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686951270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09457116574048996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686966688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09790826588869095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686981887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079886183142662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686997391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025884598493576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784687012884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08729563653469086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed490920051.log b/recommendation_v4/rcp_logs/gbs_8192/seed490920051.log new file mode 100644 index 000000000..39c5a3402 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed490920051.log @@ -0,0 +1,681 @@ +:::MLLOG {"namespace": "", "time_ms": 1784939253774, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784939253774, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784939275317, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "seed", "value": 490920051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784939275319, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784939275372, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784939275373, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784940155789, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784940155790, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784940156195, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784940331615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859927654266357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784940343626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922929763793945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940356034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384446769952774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940368533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919734954833984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940381248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388709843158722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940393804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887163996696472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940406531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390223354101181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940419279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387975513935089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940432166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13877074420452118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940444910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875946402549744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940458059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866257667541504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940471063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863620162010193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940484055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387030929327011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940496968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381441056728363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940496968, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940496969, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940584073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5020152926445007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940584074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940584074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940596793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13923002779483795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940609706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884514570236206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940623114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862669467926025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940635915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13878870010375977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940648964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13883745670318604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940662721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890577852725983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940675730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871516287326813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940688970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387302130460739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940702532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859783113002777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940715797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873599469661713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940729624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869552314281464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940743168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385280191898346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940756643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860180974006653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940770691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861210644245148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940770692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940770692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940855955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502264678478241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940855956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940855957, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940869292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382685750722885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940882771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384611427783966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940896256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810458779335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940910044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769923150539398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940923613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849398493766785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940936937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836857676506042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940950383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378992646932602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940964382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773760199546814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940977986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755056262016296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940991327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797055184841156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941004841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780245184898376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941018177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13800355792045593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941032306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379319578409195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941045726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379643976688385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941045727, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941045727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941131207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5026192665100098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941131208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941131208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941144455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731510937213898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941158113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769763708114624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941171641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372629851102829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941186028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376173049211502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941199846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376572996377945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941213715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379343718290329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941227463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721320033073425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941240979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766321539878845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941255126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372668594121933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941269085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854695856571198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941283014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369173675775528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941296835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803532719612122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941310689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581210374832153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941324866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371694803237915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941324867, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941324867, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941409149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030995011329651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941409150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941409150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941422561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644614815711975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941436228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602764904499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941449495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376180648803711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941463298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505932688713074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941477166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682569563388824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941490705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417525589466095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941504665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585898280143738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941518259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557440042495728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941531937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377052217721939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941545462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552631437778473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941559061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14621542394161224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941573098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581274449825287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941586723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431517779827118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941600390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377854198217392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941600391, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941600391, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941683852, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.503790557384491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941683853, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941683853, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941697303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355687826871872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941710806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139437735080719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941724397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617487251758575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941737961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373951613903046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941751726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519012928009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941765609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884687423706055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941779288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370142877101898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941793047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326144218444824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941806926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363991677761078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941820360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196435570716858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941834207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158074021339417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941847778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484467566013336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941861607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224339485168457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941875752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385957896709442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941875753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941875753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941958477, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046721696853638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941958478, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941958478, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941972312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387654721736908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941986088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364115357398987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941999905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358756721019745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942013419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515157997608185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942026955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771654665470123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942040525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12823735177516937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942053757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13535527884960175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942067324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12978526949882507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942080996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362035155296326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942095044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295762240886688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942108751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359582424163818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942122815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445577025413513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942136897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542531430721283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942150632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351979851722717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942150632, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942150632, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942233858, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058847665786743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942233859, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942233859, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942247865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327456682920456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942261662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378049433231354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942275637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356118768453598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942289961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356043666601181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942304076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13943403959274292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942317978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336984813213348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942332505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529184460639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942346266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330376148223877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942360182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332497000694275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942374187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193926215171814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942387903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565626740455627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942402057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335052251815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942415743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321876525878906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942429363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13056045770645142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942429363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942429364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942514834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5072068572044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942514835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942514835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942529390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376999467611313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942543421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347711682319641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942557632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550856709480286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942572020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138516247272491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942586115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366322219371796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942600452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341938078403473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942614854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136962890625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942629188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339327096939087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942643424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275788724422455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942657405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963812053203583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942670917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769916653633118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942684598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295896351337433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942698397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955772876739502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942711937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12491875886917114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942711938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942711938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942795121, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086705088615417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942795122, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942795122, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942809231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119040429592133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942823191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582536578178406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942837210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295860588550568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942851443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037991523742676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942865239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128158837556839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942879262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1267557442188263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942894058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12895750999450684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942908066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322495937347412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942922194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557016849517822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942936363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249459981918335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942950214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286659687757492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942964116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445454835891724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942978264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308278888463974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942992826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12906292080879211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942992826, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942992827, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943074925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.510524332523346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943074925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943074926, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943089320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304185688495636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943103379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291043609380722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943116944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314157724380493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943131275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12368892878293991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943145451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959475815296173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943159193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255604326725006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943173178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002286851406097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943187108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163224816322327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943200663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357278823852539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943214764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734115660190582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943228694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12365807592868805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943242920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330617010593414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943257122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12560613453388214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943271242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880557775497437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943271242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943271243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943354047, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5125288367271423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943354048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943354048, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943367845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654133677482605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943382062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656802475452423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943395685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555662333965302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943409250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13999120891094208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943422974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668790459632874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943436760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823586702346802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943450505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12724298238754272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943464254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601734697818756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943478036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1419983059167862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943492140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410660445690155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943506087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12724047899246216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943520010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857811152935028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943533737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285255432128906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943546928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149696588516235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943546928, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943546928, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943629434, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5145210027694702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943629434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943629435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943644363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14137986302375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943659011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14246876537799835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943673146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514699041843414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943687470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13963519036769867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943702105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1447536051273346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943716149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14091549813747406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943730528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12596113979816437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943744866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375715434551239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943759381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317177712917328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943773865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600246608257294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943788335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771162927150726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943802530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302433490753174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943817057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440704345703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943831383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630349934101105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943831384, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943831384, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943913941, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5166200995445251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943913942, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943913942, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943927753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841576874256134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943941848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13820961117744446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943956155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178957998752594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943970329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329849511384964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943984383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336788684129715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943998730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968269526958466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944012964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323932260274887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944027526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411522924900055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944041375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1407037079334259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944055848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13498209416866302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944069840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12766315042972565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944083622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13000017404556274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944097553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339883804321289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944111663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127712219953537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944111663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944111664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944194263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5194805860519409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944194264, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944194264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944208239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195665180683136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944222249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366698145866394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944236321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327474892139435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944250328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204774260520935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944264516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399213552474976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944278591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342995941638947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944293157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384504616260529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944307357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256260752677917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944321286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13976043462753296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944335776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13172395527362823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944349710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870317697525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944363321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107652962207794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944376984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222266733646393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944390782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13580107688903809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944390782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944390783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944472922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5234965682029724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944472922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944472923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944486926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353800773620605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944500649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1228894367814064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944514306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029327988624573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944528104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13008470833301544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944541790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350998729467392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944555962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13509775698184967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944569660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555876910686493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944583884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12396443635225296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944597836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099916279315948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944611454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309293955564499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944626100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13011713325977325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944640129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329226791858673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944654284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063709437847137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944668307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995073199272156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944668308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944668310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944750968, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5288981795310974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944750969, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944750969, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944765101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757109642028809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944779316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216263055801392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944793470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321591913700104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944807637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302211731672287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944821856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12677380442619324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944835941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318737119436264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944850424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356024146080017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944864778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257235288619995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944879029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311214715242386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944893369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867045938968658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944907595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335839331150055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944921299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723935186862946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944935893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156352937221527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944949822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309980928897858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944949823, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944949823, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945033013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5361767411231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945033014, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945033014, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945047042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309022307395935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945060933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489191234111786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945074844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376710832118988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945088777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13286380469799042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945102695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269020438194275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945116669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12915928661823273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945130576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584395706653595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945144448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12361082434654236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945158599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579653203487396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945172835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708479702472687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945186976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436158001422882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945201777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12678682804107666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945215997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130541130900383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945229679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341818958520889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945229680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945229680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945312601, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5459666848182678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945312602, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945312602, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945327112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13205109536647797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945341515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232262432575226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945355967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333199441432953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945370310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13743942975997925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945384618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594581186771393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945398945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129186630249023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945413250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12691408395767212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945427573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472893834114075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945441724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408307939767838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945456310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999001145362854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945470733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204436004161835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945485352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342787325382233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945499996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1397857964038849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945514460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605241477489471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945514461, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945514461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945595282, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.559440016746521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945595283, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945595283, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945610251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360355019569397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945624837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081102073192596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945639014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13203072547912598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945653580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836769759655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945668178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12956948578357697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945682970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782525062561035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945697052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342705488204956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945711429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367308497428894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945725627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440099358558655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945739981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040991127490997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945754293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1397962123155594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945768522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402299582958221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945782785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824536859989166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945797242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297873735427856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945797242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945797243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945878053, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.573367178440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945878054, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945878055, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945892637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.141582652926445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945906911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215243816375732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945921747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331217736005783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945936077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019666075706482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945950900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500477373600006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945965799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265404105186462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945980551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333765983581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945995404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134675070643425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946010062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12997440993785858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946024928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314457356929779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946039703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324947625398636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946054534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364191174507141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946069048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135491743683815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946083917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666030764579773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946083917, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946083918, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946166819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5862302184104919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946166820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946166821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946181436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307370811700821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946196332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13439787924289703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946210970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354723155498505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946226139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121718168258667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946240798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13617612421512604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946255250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284111589193344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946270212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327780932188034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946284359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12237422913312912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946298590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12946096062660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946312812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059017062187195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946327101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992849946022034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946341114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129959836602211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946355193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777674198150635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946369226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296989470720291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946369227, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946369227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946452218, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5953759551048279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946452219, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946452219, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946465929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12088421732187271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946479968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12525475025177002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946493915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12787780165672302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946508002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321086585521698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946522476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308131068944931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946536673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203966736793518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946551268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287183314561844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946565808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077057898044586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946580225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555549085140228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946594135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12672576308250427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946609057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317158341407776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946623544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659713834524155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946638050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965584009885788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946652506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366268783807755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946652506, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946652507, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946734575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6048654317855835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946734576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946734576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946748952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11990294605493546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946762957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12603460252285004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946777231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268959641456604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946791459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298450231552124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946805606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11968789994716644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946819921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323359191417694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946834010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276135891675949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946848086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976123601198196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946862694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12544521689414978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946877130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12519942224025726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946891320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11692162603139877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946905976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047480583190918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946920286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12569962441921234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946934580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12795472145080566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946934581, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946934581, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947018330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6133107542991638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947018331, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947018331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947032101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12629376351833344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947047063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11721578985452652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947061378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12087432295084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947075799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12000866234302521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947090084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12277712672948837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947104568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12503793835639954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947118956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431129276752472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947133636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12183865159749985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947148282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12372242659330368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947162928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12255614250898361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947177351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453165858983994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947191916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12197306752204895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947206633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804207623004913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947221166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296830862760544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947221166, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947221167, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947304641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6203591227531433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947304642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947304642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947319444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542637646198273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947334189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296132206916809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947348673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270565390586853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947363553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12081103771924973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947378138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12262531369924545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947392602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516997963190079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947406584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819484829902649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947421458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12068776041269302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947435499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11924299597740173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947449615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149261146783829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947463791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965543031692505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947477851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12075656652450562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947491873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12443070113658905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947506000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601381003856659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947506001, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947506001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947591600, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6333673000335693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947591601, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947591601, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947605663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816348135471344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947619591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012353003025055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947633354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986814647912979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947647219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282025277614594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947661396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859400033950806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947675647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12146082520484924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947689942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11573119461536407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947704550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11992179602384567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947718744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133904218673706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947733640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199394166469574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947748183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260378360748291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947762739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269273608922958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947777098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888693481683731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947791811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569935083389282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947791812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947791812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947877458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6575143933296204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947877459, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947877459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947891969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043797969818115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947906793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1190786361694336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947921597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12286657094955444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947936240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542796552181244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947950728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407865583896637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947965032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177515909075737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947979553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12003759294748306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947994258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195526048541069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948008645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830331593751907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948023197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179245188832283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948037625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510533839464188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948051902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674662679433823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948066177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328354477882385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948081048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546315789222717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948081048, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948081049, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948164545, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6901081204414368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948164546, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948164546, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948179165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.124729685485363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948193864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758358031511307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948207989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703549325466156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948222902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12277992814779282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948237353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939943045377731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948251728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176435649394989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948265642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11312877386808395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948280374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894815623760223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948295003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098593473434448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948310119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595210433006287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948324903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12410631030797958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948339383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527349054813385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948354178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12734445929527283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948368672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270029097795486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948368672, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948368673, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948452622, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7417874932289124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948452623, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948452624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948467361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607112526893616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948481789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461179703474045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948496231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164291501045227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948511002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352358758449554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948525376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11882325261831284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948540060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11846956610679626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948555007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106776982545853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948569989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12064328044652939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948584840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477416753768921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948600136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284227669239044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948614974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576158553361893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948629584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597450822591782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948644112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12481151521205902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948659365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584531515836716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948659366, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948659366, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948743632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7530791759490967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948743632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948743633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948758375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12181363254785538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948773241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955328285694122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948787833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133410632610321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948802453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451572179794312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948816972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882709175348282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948831643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11549261957406998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948846353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755172908306122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948860706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164619028568268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948875353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918166488409042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948889686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851111263036728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948904615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1229882463812828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948919509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070166677236557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948934324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12040485441684723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948949563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11985700577497482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948949564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784948949564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949033561, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7578440308570862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949033562, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949033562, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949048565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11906713992357254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949063115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11820442229509354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949077778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537687480449677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949092840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158917099237442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949107400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10345382988452911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949122011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392404347658157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949136641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11921677738428116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949151260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503717303276062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949165862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448352992534637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949180393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286813229322433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949194738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653586685657501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148856580257416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949223835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125162243843079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949238284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581775546073914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949238328, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949238329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949320710, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7610298991203308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949320711, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949320711, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949335278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104971744120121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949350060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10709669440984726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949364826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528096348047256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949379318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000335961580276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949393985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734222829341888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949408957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500781774520874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949423629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346275359392166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949438105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294480413198471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949452883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656227707862854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949467374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087504029273987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949482053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076527297496796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949496779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396826803684235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949511180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678533464670181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949525705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587577521800995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949525706, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949525706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949608626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637522220611572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949608627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949608627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949623124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061321422457695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949637904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322446167469025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949652494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0960247740149498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949667006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410737246274948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949681516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261530965566635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949695817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462532937526703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949710379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08855941891670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949724761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826919227838516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949739373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10135950148105621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949754272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724327504634857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949769115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705932229757309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949783807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969587624073029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949799024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995212197303772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949814165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702421516180038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949814165, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949814166, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949897996, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7646704316139221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949897997, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949897997, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949912908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594580858945847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949927865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218393594026566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949942117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454435646533966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949957287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636983066797256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949971418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09930862486362457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949985953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233715921640396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950000309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647007077932358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950014851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825293511152267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950029121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09694813936948776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950043733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844819247722626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950058093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044066995382309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950072333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710715502500534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950086854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849037021398544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950101430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279083788394928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950101431, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950101431, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed537267118.log b/recommendation_v4/rcp_logs/gbs_8192/seed537267118.log new file mode 100644 index 000000000..92609f271 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed537267118.log @@ -0,0 +1,1870 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651273394, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651273395, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651294763, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651294767, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651294767, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651294767, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "seed", "value": 537267118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651294768, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651294771, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651294771, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784651893648, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784651893649, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784651894026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652001264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905291259288788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652013373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383819878101349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652025915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865436613559723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652038522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13811571896076202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652051532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861262798309326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652064378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852612674236298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652077556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826708495616913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652090575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886496424674988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652103837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13849292695522308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652117135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384684443473816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652130203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387362778186798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652143386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861554861068726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652156898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842080533504486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652170071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1397927850484848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652170072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652170073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652255806, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5013150572776794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652255808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652255808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652268605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825581967830658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652281685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812769949436188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652295135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838350772857666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652308356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378079205751419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652321756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786821067333221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652335137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13765624165534973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652348165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790643215179443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652361388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778048753738403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652374929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13740061223506927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652388315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795824348926544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652401839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833366334438324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652415440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835908472537994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652428807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782469928264618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652442560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833220303058624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652442561, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652442562, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652526733, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5015504360198975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652526734, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652526734, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652540300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749249279499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652553973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138078972697258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652567665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668853044509888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652581400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361338049173355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652594836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777515292167664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652608285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801129162311554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652621758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712751865386963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652635648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654927909374237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652649317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630004227161407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652662794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747747242450714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652676518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13692618906497955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652689980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749808073043823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652703985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373593658208847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652717592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13750667870044708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652717594, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652717595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652799485, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5018365383148193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652799486, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652799487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652812723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640303909778595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652826311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371050924062729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652839730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645507395267487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652853828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687530159950256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652867550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715152442455292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652881450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759849965572357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652895259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605187833309174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652908951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726723194122314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652922923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626965880393982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652936820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845643401145935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652950543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359703689813614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652963983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761208951473236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652977584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349330097436905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652991375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637356460094452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652991375, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784652991376, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653072319, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022042393684387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653072320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653072320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653085832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582539558410645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653099446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351640224456787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653112701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794422149658203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653126357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403582572937012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653140266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644272089004517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653153837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271409273147583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653167392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13520798087120056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653180946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502292335033417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653194606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786080479621887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653208051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477793335914612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653221541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14784717559814453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653235437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524438440799713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653248895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334683895111084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653262307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766011595726013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653262308, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653262309, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653343538, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025769472122192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653343539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653343539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653357242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493965566158295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653370867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13988468050956726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653384579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556444644927979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653398297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13271233439445496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653412119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458006083965302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653426196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394202560186386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653440014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308864831924438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653453707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13265173137187958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653467258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299515843391418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653480521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311027854681015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653494096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058887422084808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653507276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347285658121109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653520774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159601390361786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653534515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905005156993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653534516, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653534516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653616707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030032396316528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653616708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653616709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653630383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335202157497406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653643926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313370943069458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653657768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585060834884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653671522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348785161972046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653685321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137860506772995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653699113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272876113653183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653712595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521234691143036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653726413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891434133052826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653740637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324779272079468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653754981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229750096797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653768905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341619074344635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653782562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443754613399506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653796677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542264699935913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653810602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336294889450073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653810602, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653810603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653892732, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037084221839905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653892733, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653892733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653906693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216347992420197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653920579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337888240814209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653934611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548748195171356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653948984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552838563919067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653963156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13980352878570557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653977048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330888271331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653991674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521912693977356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654005426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342374563217163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654019226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308016955852509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654033307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13165520131587982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654047082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573472201824188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654061399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366791605949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654075244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312064111232758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654089007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049447536468506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654089008, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654089008, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654173626, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046871900558472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654173627, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654173627, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654187989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774473965168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654202143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487617671489716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654216263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355314552783966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654230357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13109689950942993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654244362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365577161312103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654258525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338740587234497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654272976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718882203102112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654287296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13403812050819397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654301386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13281309604644775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654315261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12966188788414001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654328828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12761668860912323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654342483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330430805683136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654356237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292903572320938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654369800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.124602772295475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654369800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654369800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654453627, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5057211518287659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654453628, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654453628, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654467549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117800652980804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654481446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360829472541809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654495756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310366868972778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654510091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13030661642551422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654523924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812383472919464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654537698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265401542186737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654551976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890461087226868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654565931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13226529955863953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654579962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561370968818665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654594050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12479511648416519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654607995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877395749092102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654621820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134392648935318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654635977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088801503181458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654650463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12928347289562225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654650464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654650465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654733573, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5070603489875793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654733574, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654733575, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654747931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304352730512619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654761934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12905968725681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654775518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326068222522736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654789414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384738773107529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654803414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12939174473285675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654816911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1255599856376648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654830432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13010071218013763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654844153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12183447182178497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654857514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13567152619361877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654871437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13719011843204498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654885149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12381488084793091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654899407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324212491512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654913510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256517916917801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654927391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290084719657898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654927392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784654927392, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655010741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086907744407654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655010742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655010742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655024143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365588903427124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655038480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365814507007599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655052506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358100026845932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655066452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14007684588432312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655080616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670232892036438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655094619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834303617477417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655108556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12762440741062164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655122461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12627626955509186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655136469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1421617865562439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655150945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14100994169712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655165283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12731561064720154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655179540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12837907671928406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655193645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276323676109314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313244104385376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207231, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655207232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655290781, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5105935335159302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655290782, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655290782, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655305114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1414000689983368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655319343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14230719208717346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655333153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551203906536102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655347167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13926614820957184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655361564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14475174248218536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655375584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14106833934783936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655389830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12587971985340118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655404035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375739872455597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655418338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13180091977119446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655432648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362384855747223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655446949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793636858463287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655460929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304756581783295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655475728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443680107593536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655490452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662664592266083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655490452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655490453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655573903, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5126215815544128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655573904, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655573904, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655587886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387251317501068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655602465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383458375930786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655617160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182488083839417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655631669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309255242347717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655646186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372772932052612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655660935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13967831432819366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655675629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327541470527649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655690480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14136695861816406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655704579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1408742070198059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655718967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511493802070618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655733195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277177333831787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655747060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006910681724548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655761127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419099152088165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655775199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12776371836662292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655775200, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655775201, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655856564, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5150377750396729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655856565, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655856565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655870514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13209843635559082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655884448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693247735500336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655898401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333790421485901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655912331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13187964260578156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655926491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390159606933594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655940526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341177999973297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655954965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394001126289368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655969128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328214854001999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655983088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399206668138504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655997631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13204075396060944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656012025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12894731760025024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656025958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117629289627075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656040026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324034035205841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656054101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604873418807983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656054102, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656054102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656137096, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5182050466537476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656137096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656137097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656151276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334814578294754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656165328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303140759468079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656179338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055171072483063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656193403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304926872253418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656207379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514043390750885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656221905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536594808101654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656236072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536381721496582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656250728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12416739016771317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656264958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312824934720993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656278707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312505602836609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656292978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13021351397037506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656306963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328921794891357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656321105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094472885131836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656334891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13024933636188507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656334892, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656334892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656417161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5225784778594971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656417164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656417164, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656431252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775016367435455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656445360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324005424976349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656459371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335562914609909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656473431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130352184176445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656487435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272180825471878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656501355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319679617881775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656515769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357017457485199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656530097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279806077480316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656544402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144567608833313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656558780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391756534576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656573157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370825350284576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656587023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375160664319992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656601364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318248212337494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656615590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339447975158691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656615591, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656615591, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656696734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5287758111953735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656696735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656696736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656710936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312011182308197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656725088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504961133003235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656739439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785243034362793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656753788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330303132534027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656767920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736664712429047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656782104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922877073287964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656796096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12596295773983002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656810004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402292340993881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656824406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13631075620651245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656838946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735300302505493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656853239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346355825662613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656868117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269901990890503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656882668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308128982782364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656896684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12367988377809525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656896685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656896685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656980876, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5378608107566833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656980878, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656980878, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656995240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238655030727386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657009492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260112702846527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657023854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338382512331009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657038086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777533173561096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657052434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638554513454437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657066876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142633438110352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657081195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717366218566895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657095602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503398001194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657109799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12451492249965668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657124364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304546594619751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657138864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232453167438507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657153520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391444087028503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657168220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14027461409568787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657182681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613636791706085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657182682, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657182683, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657264825, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5516666173934937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657264827, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657264827, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657279852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653331995010376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657294480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144652545452118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657308736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323525607585907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657322998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390550434589386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657337808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298941820859909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657352974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384761780500412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657367465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502159714698792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657382017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723310828208923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657396460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134572833776474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657411134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038048148155212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657425698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1403605043888092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657440268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347608119249344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657454847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289958357810974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657469467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316909968852997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657469468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657469469, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657551653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5675350427627563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657551654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657551654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657566546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14188151061534882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657581101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235323131084442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657596326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133346289396286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657610972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026738166809082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657625960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530735671520233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657640623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334136426448822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657655465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390342891216278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657670512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558954000473022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657685411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090038299560547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657700576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242791593074799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657715499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327843964099884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657730492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13697364926338196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657745190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361313909292221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657760140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715185225009918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657760141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657760141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657843041, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5848743319511414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657843042, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657843042, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657857775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142886757850647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657872794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522754609584808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657887526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409875333309174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657902773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192200660705566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657917598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371985375881195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657932319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12931403517723083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657947303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133403941988945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657961681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341707944869995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657976279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044756650924683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657991023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111522793769836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658005722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309221088886261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658020375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13082937896251678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658035069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12877953052520752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658049695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13015419244766235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658049695, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658049696, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658130604, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5996133685112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658130605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658130605, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658145056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12181005626916885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658159674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266557276248932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658174166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856416404247284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658188813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325235664844513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658203841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150878250598907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658218456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12118998169898987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658233476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12994740903377533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658248403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13148283958435059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658263266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671283423900604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658277529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12732014060020447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658292290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250099122524261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658306803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170162633061409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658321365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12050482630729675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658336128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12465347349643707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658336128, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658336129, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658417330, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6135889291763306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658417330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658417331, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658431976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12068549543619156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658446457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12670598924160004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658461072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1279887855052948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658475732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13151425123214722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658490199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058936804533005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658504794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341048777103424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658519138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12890098989009857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658533474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12033837288618088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658547917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649273872375488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658562543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126371830701828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658576948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750021576881409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658592008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13062484562397003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658606621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261119544506073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658621021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12764962017536163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658621022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658621022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658703418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6267098784446716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658703419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658703419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658717399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651419639587402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658732027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176154688000679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658746614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12102960050106049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658761479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995167285203934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658775945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321807444095612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658790480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12511321902275085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658805047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470798403024673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658819701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12150394916534424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658834220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408538907766342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658848743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12227829545736313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658863139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433691531419754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658877647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12208706885576248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658891990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825183987617493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658906661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12306025624275208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658906661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658906662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658989203, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6379542946815491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658989203, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658989204, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659004384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11502925306558609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659019205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257729679346085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659033807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699761986732483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659048760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021943181753159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659063662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12229187786579132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659078151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1146773099899292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659092100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813510209321976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659106627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12070698291063309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659120864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11855504661798477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659135074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091650277376175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659149238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183241605758667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659163383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208546832203865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659177557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12334374338388443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659191791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11572325974702835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659191791, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659191792, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659274372, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6520432829856873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659274373, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659274373, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659288458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11744555830955505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659302521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928478837013245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659316446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844939947128296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659330327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192812025547028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659344247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663960129022598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659358291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12116225808858871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659372666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377149820327759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659387549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11914889514446259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659401893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10044839978218079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659416504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11946272850036621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659431109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180496215820312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659445506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170543730258942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659459685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763774812221527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659473945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583017557859421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659473946, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659473946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659557432, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6697787642478943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659557433, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659557433, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659571753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11904582381248474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659586480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189282014966011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659601162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12191496044397354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659615549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360098630189896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659630102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282636225223541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659644427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593680828809738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659658832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872338503599167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659673250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11904513835906982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659687430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672628879547119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659701563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773096024990082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659715879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114909328520298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659729897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470399051904678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659744028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169275641441345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659758724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12516385316848755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659758724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659758725, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659843297, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6870991587638855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659843298, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659843298, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659858002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12403938919305801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659872695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607769131660461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659887140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163368746638298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659902093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12071496248245239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659916610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086399108171463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659931310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11089086532592773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659945378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261677742004395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659959892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084282398223877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659974446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121903359889984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659989474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608056724071503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660004182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12328866124153137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660018711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485080420970917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660033724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275276243686676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660048455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234962195158005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660048456, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660048456, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660132153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7157189846038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660132154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660132154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660146985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693756818771362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660161543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385948956012726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660176014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162511557340622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660190935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292923986911774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660205723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12001514434814453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660220760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11886285990476608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660236116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12130503356456757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660251390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058158218860626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660266389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504197865724564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660281878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340747028589249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660296808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622749269008636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660311453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587648093700409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660326019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607769668102264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660341085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590025573968887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660341086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660341086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660424566, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7337918281555176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660424567, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660424567, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660439266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12271557748317719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660454336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983563750982285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660469230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392635852098465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660484355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115829698741436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660499133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927470028400421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660513932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598075181245804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660528747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11891163885593414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660543134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213206499814987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660557813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10981077700853348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660572349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865087807178497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660587396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12188133597373962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660602502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652673244476318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660617344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1217101439833641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660632738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12093881517648697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660632739, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660632739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660715578, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7467658519744873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660715579, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660715579, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660730615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995065212249756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660745386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1197282075881958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660760128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11611475795507431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660774936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1170867457985878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660789563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368441790342331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660804347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520084947347641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660819074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12090286612510681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660833658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510937660932541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660848229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046375185251236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660862667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039443388581276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660876763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830876231193542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660891086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325961351394653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660905352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11384862661361694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660919678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696934700012207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660919719, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660919720, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661002888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.755730390548706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661002889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661002890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661017409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551406443119049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661032242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836630314588547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661047328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690673232078552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661062122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245717108249664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661077153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829795151948929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661092350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11732516437768936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661107113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403567343950272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661121653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506544798612595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661136091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11743248254060745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661150547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252182722091675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661164810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169632732868195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661179299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561254411935806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661193515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819903761148453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661207864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729663819074631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661207865, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661207866, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661292958, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7603189945220947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661292959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661292959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661307340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085595116019249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661321964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449199914932251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661336604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09783316403627396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661351186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482700169086456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661365663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435598880052567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661379977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656856000423431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661394532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09008070826530457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661408806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966043919324875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661423082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035551205277443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661437608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816447973251343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661452188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817869007587433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661466459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096599161624908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661481254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283384472131729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661495944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850118100643158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661495945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661495945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661579210, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637795209884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661579211, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661579211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661593679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639527976512909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661608093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219251900911331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661622012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443322360515594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661636708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831297934055328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661650972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032288730144501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661665563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584508627653122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661680014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877303779125214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661694637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985640436410904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661709121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949423372745514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661723761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033382266759872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661738205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567264258861542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661752675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913330316543579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661766963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087067499756813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661781417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363359540700912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661781418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661781418, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661863496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7643066644668579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661863497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661863497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661878076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039556562900543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661892286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140808671712875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661906807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431034117937088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661921548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661078244447708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661936528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290248483419418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661951580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363186687231064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661966477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108590871095657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661981310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646922141313553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661996325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823135077953339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662011204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656394064426422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662026232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282693594694138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662040510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0955958291888237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662055477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09870600700378418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662069803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762746632099152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662069803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662069804, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662153840, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659448385238647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662153841, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662153841, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662168050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09175947308540344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662182386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504128783941269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662196607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022251099348068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662211058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098680779337883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662225206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074241995811462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662239462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166910827159882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662253826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815622121095657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662267794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057554185390472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662281919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549072176218033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662295986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059456467628479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662310301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871917754411697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662324290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441464185714722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662338397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09744545817375183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662352680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649677366018295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662352681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662352681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662435974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766512930393219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662435975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662435975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662450556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458602011203766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662465284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457268357276917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662479611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569605976343155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662494159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392570495605469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662508768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837830066680908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662523168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10056566447019577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662537917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682579129934311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662552065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09586073458194733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662566709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080840528011322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662581361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911634564399719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662596073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616100370883942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662611013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11440127342939377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662625711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620751768350601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662640437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507882922887802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662640438, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662640438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662723721, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678120136260986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662723722, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662723722, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662738233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131610184907913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662753037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617577821016312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662767571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872867703437805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662782404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024293527007103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662797016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09116002917289734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662811703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834719449281693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662826065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446259379386902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662840840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744815319776535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662855233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802064090967178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662869710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061800479888916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662884742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306782811880112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662899647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000417172908783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662914552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953826457262039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662929355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11205697804689407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662929355, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662929356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663011741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679907083511353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663011742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663011743, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663026731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082443818449974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663041684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165593773126602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663056529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058223992586136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663071604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09161302447319031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663086208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473284661769867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663100792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068863794207573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663116025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109202355146408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663130760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179428547620773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663145859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12224256247282028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663160648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556275397539139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663175522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175458133220673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663190525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12328340113162994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663205332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013885796070099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663220069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126118153333664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663220069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663220070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663301973, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687667012214661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663301974, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663301975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663316737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454314202070236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663331637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894288659095764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663346269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904433578252792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663361482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395601391792297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663376316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09701276570558548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663391087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302808672189713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663406401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665623098611832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663421468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609465837478638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663436405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682891845703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663450859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994146317243576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663465744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553928792476654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663480582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818720608949661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663495116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11710181832313538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663509636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792446881532669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663509637, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663509637, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663593364, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688585519790649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663593365, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663593365, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663608165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11381227523088455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663623379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524175852537155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663638567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057521402835846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663654065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779818683862686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663669248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447061598300934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663684568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923524737358093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663699900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127125546336174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663714823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12121661007404327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663729854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626705527305603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663745263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053109169006348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663760242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829618573188782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663775384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873909294605255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663790509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11556925624608994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663805696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142469123005867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663805697, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663805698, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663886576, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698290944099426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663886577, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663886577, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663901928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12054620683193207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663917285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747353732585907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663932211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082199439406395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663947645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765772312879562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663962983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065156012773514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663978414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963744670152664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663993467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421643942594528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664008318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784246772527695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664022962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626571625471115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664037396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047345608472824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664052336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458473861217499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664066876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693833440542221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664081393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417400300502777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664095665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10106176882982254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664095666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664095666, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664177383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707108855247498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664177384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664177384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664191745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978298634290695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664206255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111866146326065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664220674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962742269039154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664235070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864695906639099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664249469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540719866752625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664263819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521764308214188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664278234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479560494422913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664292697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049768403172493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664307005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150916874408722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664321461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12067445367574692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664335978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776181310415268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664350492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127206146717072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664365326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053417086601257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664379689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034996598958969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664379690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664379690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664462078, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708470225334167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664462078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664462079, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664477050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075176298618317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664491913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09679156541824341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664506653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506570339202881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664521346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336899757385254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664535705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861444473266602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664550328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515379905700684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664565234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670946538448334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664579951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206580698490143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664595117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498671978712082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664610190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767506808042526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664625070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546337068080902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664639966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11473076045513153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664654574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647991299629211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664669388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10216576606035233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664669389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664669389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664752170, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713707089424133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664752171, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664752171, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664767146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928009659051895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664782203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09409144520759583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664797080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09622393548488617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664812125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09701599180698395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664826913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711576253175735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664841892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0950046181678772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664856739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021603852510452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664871529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09694884717464447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664886382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295744240283966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664901071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861501097679138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664916241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483536869287491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664931044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335119068622589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664945791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423522979021072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664961064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126511543989182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664961064, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664961065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665046756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717000842094421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665046757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665046757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665061491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845655411481857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665076297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071082279086113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665091253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101929172873497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665105832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120666772127151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665120345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042130202054977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665135210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786592215299606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665149797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11078642308712006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665164434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604935139417648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665179228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026066467165947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665193650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012057289481163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665207990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995919466018677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665222436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235434770584106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665236870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092282310128212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665251170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438706517219543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665251171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665251171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665337210, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718310356140137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665337211, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665337211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665351767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022753268480301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665366452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694991052150726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665381043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09113290905952454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665395430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10333118587732315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665410286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677170753479004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665424805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077955961227417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665439327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063235253095627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665453902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308889299631119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665468795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019604116678238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665483646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410460084676743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665498641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281877219676971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665513497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081809401512146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665528768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544139891862869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665543893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880322754383087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665543895, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665543895, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665630340, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722253799438477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665630341, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665630341, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665645212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368102043867111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665660106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317471623420715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665674756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038704514503479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665689049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029072180390358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665704062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072206124663353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665718622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145393759012222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665733316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09832171350717545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665748345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223356634378433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665763186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198436468839645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665778137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523805022239685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665792786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414654016494751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665807436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258986055850983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665822323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434655100107193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665837015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640380531549454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665837016, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665837017, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665923402, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726708650588989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665923403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665923403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665938088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863188654184341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665952763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016484946012497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665967092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039639338850975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665981666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117370635271072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665996339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455064475536346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666010776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11332662403583527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666025382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158169269561768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666039741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052798479795456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666054510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132238432765007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666069661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611864924430847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666084383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339532792568207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666098985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461749136447906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666113895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568413138389587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666128779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875227510929108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666128780, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666128780, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666215114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730655074119568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666215115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666215115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666230022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219410806894302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666244941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940332174301147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666259689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602445900440216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666273903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037476569414139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666288912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828374326229095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666303646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10945728421211243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666318496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752979665994644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666333607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538208276033401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666348501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378853023052216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666363289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004980057477951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666378083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10392866283655167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666392795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028134822845459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666407402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561169683933258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666421971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746919363737106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666421971, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666421972, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666504559, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731917500495911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666504560, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666504560, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666519349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198807716369629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666534013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078465193510056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666548999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938993096351624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666563600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486166179180145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666578085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0943354144692421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666593113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254645138978958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666607475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125099122524261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666621923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764870047569275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666636881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311269015073776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666651860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385473072528839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666666380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093599870800972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666681206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114145427942276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666696135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645928233861923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666711236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283980518579483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666711236, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666711237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666793923, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773276686668396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666793924, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666793924, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666809022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11264753341674805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666823833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921858996152878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666838436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961868613958359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666853434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729439556598663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666867750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105253010988235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666882044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717029124498367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666897201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11791053414344788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666912209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907493531703949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666927201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639586836099625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666942513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12248367816209793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666957425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714315623044968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666972434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031551659107208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666987613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500699818134308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667002611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004745215177536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667002613, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667002613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667085581, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733807563781738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667085582, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667085582, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667100751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324610561132431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667116054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941973328590393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667131232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083541065454483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667146410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693589597940445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667161597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393813252449036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667176753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11372072249650955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667192046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068851575255394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667207219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477333307266235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667222748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199472844600677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667238225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032651156187057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667253665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629619657993317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667268985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11701486259698868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667284145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730667412281036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667299687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11870327591896057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667299687, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667299688, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667384152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735241651535034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667384153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667384154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667399429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11937782913446426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667414620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853338986635208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667429833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194179505109787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667444750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137010008096695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667459693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072809025645256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667475099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11511129140853882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667490061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159142851829529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667505245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678563803434372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667520452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129024550318718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667535475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058245375752449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667550468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306238174438477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667565553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685847699642181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667580500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695557087659836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667595695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052650734782219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667595695, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667595696, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667679770, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742419838905334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667679772, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667679772, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667694766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953886806964874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667710126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11892169713973999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667725500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037251353263855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667740917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349950730800629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667756218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605124920606613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667771613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883084684610367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667787124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810153186321259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667802792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095430999994278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667818542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281441897153854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667833698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261613667011261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667849217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010776460170746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667864932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032350361347198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667880573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113646849989891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667896022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937625914812088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667896022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667896023, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667979731, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743655443191528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667979732, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667979733, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667994634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181516945362091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668009615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062642931938171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668024536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038575321435928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668039292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243108868598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668054203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600598901510239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668069444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11168890446424484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668084051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776625573635101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668098692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736948251724243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668113725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09793344885110855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668128447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169947892427444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668143063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549633204936981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668157713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496149957180023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668172624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926634073257446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668187604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11598134785890579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668187605, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668187605, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668269744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746367454528809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668269745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668269745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668284415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476525664329529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668299172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530152916908264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668314192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065707564353943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668328982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09263133257627487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668343673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366003006696701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668358223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303189605474472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668373007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250444710254669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668388057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247333884239197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668402881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735206305980682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668417826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034853607416153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668432990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909824073314667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668448336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063424497842789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668463611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730907320976257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668478753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676854848861694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668478754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668478754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668560419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747383117675781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668560420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668560420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668575343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002103239297867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668590577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331790894269943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668605481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028443276882172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668620230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09888075292110443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668635346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607608199119568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668650381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011181771755219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668665144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035495325922966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668680343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801719129085541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668695404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804269462823868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668710121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108755923807621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668725115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11076835542917252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668739935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885197877883911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668754438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071598008275032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668768873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0874006524682045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668768874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668768875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668851814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749252319335938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668851815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668851816, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668866546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09535553306341171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668881312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166112124919891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668896060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701604187488556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668910779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127848386764526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668925582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186677426099777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668940437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09036796540021896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668955213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346736758947372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668969955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09756474196910858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668984619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10387992113828659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668999439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616518557071686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669014454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055491864681244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669029632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837268084287643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669044491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580842196941376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669059567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393217951059341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669059568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669059568, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669141469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749408483505249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669141470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669141470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669156678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621662646532059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669172128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526090115308762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669187416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09527690708637238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669202691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145485401153564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669217831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09730442613363266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669232641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082779079675674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669247342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913863778114319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669262660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112395882606506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669277632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913747549057007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973455011844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669308304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620740056037903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669323743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433526337146759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669338760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942414402961731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669353785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876781493425369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669353786, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669353786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669437400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752856612205505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669437401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669437401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669452384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696763545274734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669467198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047927588224411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669481878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322121530771255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669496728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09969180077314377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669511693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106618732213974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669526626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09590014070272446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669542127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526064783334732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669556946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289575159549713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669571839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10995656251907349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669586849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09908945858478546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669601434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741455316543579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669616277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185326635837555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669631343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09088223427534103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669646550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706773400306702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669646551, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669646551, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669729801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.775657594203949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669729802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669729803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669745148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973179548978806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669760627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033995658159256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669775420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09972897917032242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669790461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960947722196579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669805591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001359075307846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669820801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218656808137894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669835851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177962481975555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669850942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054084300994873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669866132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170314460992813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669880918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221630126237869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669896017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068412125110626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669910785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10798826068639755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669925615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602875798940659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669940584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616553574800491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669940585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669940585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670023942, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760641574859619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670023943, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670023943, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670038880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572603344917297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670053688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828606247901917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670068277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655493289232254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670083210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702754557132721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670098008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265825688838959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670112470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734546184539795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670127052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741023182868958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670141845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09822960942983627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670156533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496596992015839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670171333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103910893201828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670186163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10272553563117981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670200961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969291627407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670215792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069038063287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670230608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002931296825409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670230608, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670230609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670313419, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761626839637756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670313420, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670313420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670328126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293201357126236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670342883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523776710033417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670357688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09540973603725433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670372902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09928587079048157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670387828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194390267133713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670403096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10074706375598907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670418139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722863674163818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670433350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982834547758102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670448502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883606970310211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670463537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019560158252716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670478776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020621508359909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670493425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09778377413749695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670508884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191254109144211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670523680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123016476631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670523681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670523681, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670605816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763913869857788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670605817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670605817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670620464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602010041475296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670636097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478255152702332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670651295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588279366493225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670666927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116191744804382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670682817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672806203365326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670698502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931304097175598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670713881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075752004981041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670729460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976814478635788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670744941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647989064455032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670760242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428845882415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670775532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854119807481766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670790554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334066301584244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670806060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617552697658539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670821282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09636261314153671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670821334, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670821334, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670902690, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764504551887512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670902691, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670902691, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670917985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09869398176670074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670933327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553297400474548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670948757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238863527774811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670963964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431334376335144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670979034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799982398748398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670993773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12293615937232971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671008713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873296856880188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671023669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09900733083486557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671038672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197234690189362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671053597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115211471915245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671068714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455177515745163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671083995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09913278371095657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671099422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632834583520889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671114658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037500724196434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671114707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671114707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671197233, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7770000100135803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671197234, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671197234, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671212403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09880984574556351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671227590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879492968320847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671242737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230605840682983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671258163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787409335374832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671273236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311928182840347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671287899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106879711151123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671303345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718660056591034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671318546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11990966647863388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671333675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461977869272232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671349264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859663784503937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671364773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104474738240242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671380092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030980944633484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671395425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11393123865127563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671410781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197621375322342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671410782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671410782, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671494206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774386405944824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671494208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671494208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671509575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805604606866837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671524869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032863706350327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671540312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1149178147315979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671555468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993768274784088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671570459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841679573059082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671585653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844794660806656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671601045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170120745897293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671616254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275909096002579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671631558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12047013640403748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671646992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930614173412323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671662102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868102312088013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671677357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479474067687988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671692844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297378897666931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671708013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880367457866669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671708014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671708015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671790596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.777470588684082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671790597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671790597, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671805670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448246240615845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671821030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955557018518448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671836404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375382542610168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671851903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253856122493744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671867305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077447980642319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671882402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195726990699768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671897860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117404699325562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671912638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10966477543115616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671928042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11020933091640472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671943443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09878554940223694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671958727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969100892543793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671974427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933828353881836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671989883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450435429811478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672004947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035916209220886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672004948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672004949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672089580, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776263952255249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672089581, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672089581, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672104606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305443942546844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672119750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12731507420539856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672134985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10872355103492737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672150107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234673112630844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672165265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127481609582901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672180615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101729422807693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672195954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409436166286469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672211253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135326474905014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672226231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684249550104141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672241440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869015008211136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672256957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033974170684814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672272145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255691945552826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672287726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339547485113144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672303245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627250909805298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672303246, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672303246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672386619, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777079939842224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672386620, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672386621, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672401937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09635580331087112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672417477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138806492090225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672432934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12173325568437576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672448441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718898475170135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672463865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043170839548111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672479009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765890032052994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672494290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277633160352707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672509336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129363402724266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672524461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334969311952591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672539059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785952746868134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672553870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705503076314926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672568944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115894466638565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672584177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09255003929138184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672598863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399321466684341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672598864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672598864, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672684220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776152491569519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672684221, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672684221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672699215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237597674131393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672714046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09628530591726303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672728633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390165984630585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672743281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850359499454498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672757853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967843234539032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672772895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426661372184753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672787720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819854587316513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672802479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130601242184639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672817239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980181604623795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672832271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0956762284040451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672847224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09695913642644882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672861960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09747710078954697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672876444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241592675447464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672891094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077252238988876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672891094, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672891095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672975805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775678038597107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672975806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672975806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672990769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293340682983398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673005875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322614014148712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673020781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339131206274033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673035650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734470188617706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673050440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10235617309808731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673065597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983355551958084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673080810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913494974374771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673095755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522083193063736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673110694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654790699481964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673125441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592575371265411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673140466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478770732879639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673155064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628117620944977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673169546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585664957761765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673184705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11357520520687103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673184751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673184752, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673268994, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783006429672241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673268995, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673268995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673283737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829005390405655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673298474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11165773123502731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673313565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072721853852272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673328575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748091340065002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673343148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962110966444016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673358094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214615613222122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673372988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551379829645157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673387685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691250115633011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673402371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09958139061927795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673417208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150792449712753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673432195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204659402370453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673446917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287555307149887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673461790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924736618995667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673476659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364341735839844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673476704, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673476705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673559456, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783663868904114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673559457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673559458, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673574360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09782955050468445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673589432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844257473945618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673604251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960614144802094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673619090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10229236632585526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673633886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0937352329492569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673649069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686129331588745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673664389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299418121576309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673679322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315241664648056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673694527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480692237615585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673709588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211661249399185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673725082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09477433562278748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673740383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112203598022461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673755882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230438947677612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673770921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060952603816986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673770970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673770971, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673854167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7786232233047485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673854168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673854168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673869138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09776774793863297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673884580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720999538898468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673900095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1022408977150917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673914995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503406077623367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673930148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064156353473663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673944975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842219740152359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673959983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032350569963455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673975263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846900403499603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673990592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861381143331528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674005822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847962647676468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674020837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085485875606537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674035990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09365560114383698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674050810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09577857702970505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674065749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573173314332962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674065749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674065750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674148688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778984546661377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674148689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674148689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674163720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300062596797943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674178652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379949748516083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674193567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10164198279380798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674208491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536274313926697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674223264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128010600805283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674238166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065484791994095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674253008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856766253709793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674267742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138743370771408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674282510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069948598742485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674297139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028190478682518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674311979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09798133373260498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674327207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210732161998749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674342420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963701993227005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674357388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937993228435516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674357389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674357390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674440897, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793816924095154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674440898, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674440898, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674456084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558873951435089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674471053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120101273059845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674486652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077489912509918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674501913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112103074789047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674517163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626595467329025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674532379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937550127506256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674547448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998929888010025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674562779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651737451553345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674577394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105538509786129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674591911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09413986653089523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674607612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402215272188187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674622821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119221359491348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674638154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09340885281562805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674653663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10077707469463348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674653663, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674653664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674736204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.779661238193512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674736206, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674736206, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674751753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036303639411926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674767203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382913589477539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674782610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121094435453415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674797897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072535514831543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674813143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301872342824936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674828397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202836990356445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674843729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688689351081848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674858849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928075760602951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674873917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971297174692154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674888853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741850197315216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674904066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158184707164764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674918967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09754068404436111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674934197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274063795804977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674949030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241048038005829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674949031, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674949031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675032018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7797478437423706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675032019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675032019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675047071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102015919983387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675061856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074967160820961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675076638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271261423826218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675092165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09419453889131546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675107440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299596935510635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675122505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725047439336777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675137662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519558727741241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675152788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083010792732239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675168135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10556194186210632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675183722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957034677267075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675198873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09706181287765503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675214081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720019787549973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675229624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09882110357284546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675244594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037924513220787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675244595, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675244595, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675327183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799726724624634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675327184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675327184, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675342689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980929434299469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675357831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468385368585587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675372655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09256668388843536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675387814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066402703523636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675402650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883717238903046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675417576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144675314426422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675432966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09656210243701935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675448275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416623204946518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675463289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634192824363708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675478220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09901802241802216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675493266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041516363620758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675508141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10405829548835754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675522958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11093925684690475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675538086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454667150974274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675538086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675538087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675620522, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799948453903198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675620523, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675620524, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675635482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003132164478302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675650701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494028031826019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675665608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510677844285965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675680878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817872732877731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675695932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131540894508362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675711129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717411547899246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675726083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258437693119049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675740864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117018535733223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675761764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098510980606079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675776745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526977688074112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675791746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540826618671417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675807082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260036379098892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675822391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020289957523346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675837516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834818124771118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675837517, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675837517, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675921766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7803231477737427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675921767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675921767, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675937107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042553395032883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675952087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09739439934492111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675967327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625086724758148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675982659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214453727006912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675998090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11760076880455017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676013347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09638525545597076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676028656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054893508553505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676043673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170315951108932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676059091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10108616948127747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676074217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10470834374427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676088923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065049096941948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676104393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098676547408104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676119719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09948199987411499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676135527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499764770269394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676135527, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676135528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676220058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.781006395816803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676220059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676220059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676235824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234048008918762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676251608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133856326341629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676267392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600941628217697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676283320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11987508088350296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676299163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11559200286865234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676315027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181745141744614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676331068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11805438995361328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676346817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11319378018379211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676362682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590420663356781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676378607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077098622918129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676394654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761167109012604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676410474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667447745800018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676426440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757973790168762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676441882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11685274541378021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676441883, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676441883, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676527645, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811669707298279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676527645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676527645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676543646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1122468039393425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676559359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10396813601255417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676575002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402611643075943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676590820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610335111618042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676606728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835517197847366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676622402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570959001779556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676638345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495207458734512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676654236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11431242525577545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676670455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780320852994919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676686446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045931801199913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676702518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775758117437363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676718414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135101318359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676733922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811511427164078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676749873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11596212536096573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676749874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676749875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676835932, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816504836082458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676835933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676835933, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676851264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040435194969177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676866886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128810048103333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676882333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408628523349762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676897590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10128842294216156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676913295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143864154815674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676928978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935956984758377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676944326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09869091212749481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676959615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824012756347656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676974905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159265995025635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676990037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570728033781052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677005254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449843645095825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677020512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11084305495023727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677035494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1179816722869873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677051006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274842917919159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677051006, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677051007, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677137646, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782129168510437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677137647, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677137648, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677152899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251890659332275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677168244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735400766134262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677183487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106371358036995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677198790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11632662266492844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677213947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171797662973404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677229277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776918917894363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677244937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984084963798523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677260415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897643864154816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677275909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734251141548157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677291432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916750133037567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677307165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703670769929886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677322984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162159591913223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677338522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220806300640106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677354309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191898584365845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677354309, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677354310, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677439906, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816577553749084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677439907, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677439907, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677455060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513134300708771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677470538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083178967237473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677485633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894278973340988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677501652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618061572313309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677516955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602065920829773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677532392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102405399084091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677548545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633635520935059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677564033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970672965049744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677579721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359009146690369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677595223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09363829344511032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677610834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11716305464506149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677626238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132802277803421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677641187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899957478046417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677656301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774485021829605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677656301, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677656302, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677739692, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822102308273315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677739693, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677739693, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677755157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248102992773056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677770671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09373950213193893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677786163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661147743463516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677801476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842035710811615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677816869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905250161886215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677832295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246199160814285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677847623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0957522988319397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677862913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060970276594162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677878230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595804452896118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677893418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034715920686722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677908622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09864529967308044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677924222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951204061508179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677939519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057724803686142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677954750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875418245792389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677954751, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677954751, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678036881, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7824049592018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678036882, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678036882, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678052398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584244877099991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678067750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09595076739788055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678083107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039179340004921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678098680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0956374853849411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678114235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265258699655533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678129551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115102618932724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678144849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654888302087784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678159747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959153100848198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678174783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11333131045103073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678189951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399134457111359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678204624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10297806560993195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678219953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166628658771515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678234904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10373738408088684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678250225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107989609241486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678250225, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678250226, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678333380, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829484939575195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678333382, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678333382, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678348797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052739828824997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678364044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922217577695847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678379274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256920754909515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678394162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224701464176178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678409507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09892712533473969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678424772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014966368675232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678440039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09576165676116943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678455000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918870031833649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678470263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342995822429657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678485338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877618938684464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678500401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495041310787201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678515304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745169222354889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678530520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197096318006516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678545419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09570567309856415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678545420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678545420, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678630506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833391427993774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678630508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678630508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678645614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09447591006755829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678660407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09956313669681549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678675254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09645996242761612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678689805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983590245246887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678704719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288690030574799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678720216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09543519467115402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678735237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287076979875565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678750504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824525564908981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678765473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09831804037094116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678780525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421779006719589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678795542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09287778288125992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678810856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705714672803879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678826128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525722801685333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678841603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10961825400590897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678841604, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678841604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678924740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7840125560760498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678924741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678924741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678939754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473591834306717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678954770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10058774054050446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678969749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528627038002014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678985041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311265289783478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678999743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606003016233444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679014432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467848926782608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679029390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09524332731962204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679044266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137469932436943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679059146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982033759355545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679074701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09064000099897385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679089988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000346601009369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679105001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522542893886566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679119925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359357297420502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679134891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866481810808182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679134892, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679134892, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679217814, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837282419204712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679217815, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679217815, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679232730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054497420787811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679247375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544989258050919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679262218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09761001914739609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679277072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916829317808151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679292041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996568575501442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679306743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0937655046582222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679321583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103753000497818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679336568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133164376020432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679351262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027871280908585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679366456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013643443584442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679381211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09734304249286652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679396237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986454248428345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679411327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.099483922123909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679426545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560328513383865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679426546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679426547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679508728, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836151123046875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679508729, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679508729, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679523899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118672996759415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679539181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09485578536987305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679554493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09799488633871078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679569505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10759411007165909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679584921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321011394262314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679600222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08723301440477371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679615606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617394745349884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679631040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09266380965709686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679646525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364456474781036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679661734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09514205157756805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679676623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09434378892183304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679691323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254760086536407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679706239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220698803663254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679721163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09743020683526993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679721164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679721164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679805003, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837023138999939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679805004, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679805004, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679819448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566503018140793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679834572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09491308033466339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679849782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204130411148071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679864684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020173728466034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679880109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374946892261505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679895455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10514166951179504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679910684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883693397045135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679925684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115514375269413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679940893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829773545265198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679956122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09895297139883041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679971417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103682592511177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679986350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09635252505540848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680001677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09317448735237122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680016887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847397148609161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680016887, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680016888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680102068, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7826203107833862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680102069, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680102069, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680117429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09900287538766861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222658560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680132742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1138279139995575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680147874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10543506592512131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222986240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680162995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09222284704446793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680178257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036883145570755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223313920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680193592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742192178964615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680208758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09478235244750977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223641600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680223829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527403652667999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed546672350.log b/recommendation_v4/rcp_logs/gbs_8192/seed546672350.log new file mode 100644 index 000000000..4ac87d688 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed546672350.log @@ -0,0 +1,678 @@ +:::MLLOG {"namespace": "", "time_ms": 1784930369192, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784930369192, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784930389693, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "seed", "value": 546672350, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784930389698, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784930389707, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784930389707, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784930974695, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784930974697, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784930974995, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784931186696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390204131603241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784931199265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786539435386658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931212192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865627348423004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931225397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802677392959595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931238622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383385956287384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931252022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846565783023834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931265318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381155103445053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931278824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13922083377838135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931292341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839635252952576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931305842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381588876247406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931319805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873346149921417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931333647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389465630054474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784931347407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854007422924042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931360974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14015771448612213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931360974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931360975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931445741, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4994525909423828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931445742, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931445742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784931459401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784918189048767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931473266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380368173122406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931487254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816162943840027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931500859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377783715724945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931514803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764257729053497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931529104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13747434318065643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931542952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779786229133606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931556871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760286569595337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931571039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718311488628387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931584992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758327066898346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931599134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13812220096588135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931613491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138294979929924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931627613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13753624260425568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931642090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794384896755219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931642091, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931642091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931725632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49967291951179504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931725633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931725633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784931739887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737691938877106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931754148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13809585571289062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931768324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668009638786316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931782680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357746720314026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931797053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734029233455658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931811344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803184032440186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931825531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661524653434753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931839948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13619530200958252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931854205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13620692491531372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931868400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13744208216667175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931882835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13678397238254547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931896908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373472809791565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931911732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752947747707367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931926219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736192882061005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784931926220, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784931926220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932010103, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49994543194770813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932010104, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932010104, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784932024540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615094125270844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932039111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370156854391098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932053478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623012602329254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932068345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699166476726532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932082884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370358020067215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932097743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13763421773910522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932112460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360657811164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932126920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718916475772858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932141694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652119040489197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932156596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385268270969391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932171322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357440948486328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932185911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375763714313507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932200429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474738597869873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932215040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363845318555832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932215041, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932215041, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932297744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.50034099817276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932297745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932297745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784932312068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13581424951553345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932326564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350419968366623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932340806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773706555366516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932355278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401590287685394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932369935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641871511936188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932384268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275942206382751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932398633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352444589138031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932412873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13495025038719177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932427314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790039718151093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932441667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347585767507553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932456113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14856696128845215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932470776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532550632953644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932485141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341832160949707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932499300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791024684906006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932499300, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932499301, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932583794, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5008062124252319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932583795, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932583795, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784932598314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13518068194389343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932612727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13986483216285706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932627311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357782781124115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932641650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278305530548096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932656223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478398323059082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932670959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13923342525959015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932685472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299208879470825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932699997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275980949401855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932714431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306425511837006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932728663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118581473827362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932743221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073016703128815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932757456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487932085990906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932772000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157713413238525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932786805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899488747119904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932786805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932786806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932872638, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014133453369141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932872639, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932872639, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784932887077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13349208235740662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932901369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317444920539856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932915952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13556191325187683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932930580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13489803671836853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932945140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137803316116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932959722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12744081020355225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932974037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525868952274323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784932988521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292271614074707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933003181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328766822814941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933017990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244707882404327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933032525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353879749774933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933046927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343860924243927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933062064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549068570137024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933076606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323698937892914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933076607, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933076607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933162097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502288818359375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933162098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933162098, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784933176977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323731392621994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933191721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368788361549377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933206479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356704980134964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933221462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550212979316711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933236144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968992233276367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933250607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331995725631714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933265980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355317085981369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933280859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340544700622559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933295849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312242925167084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933311188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178585469722748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933326171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584616780281067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933341306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361775875091553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933356000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339564204216003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933370584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307116597890854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933370584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933370585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933456410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034345388412476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933456411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933456411, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784933471555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788090646266937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933486260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487717509269714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933501125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354423463344574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933516136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13130652904510498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933530829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361583650112152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933545580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404113054275513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933560621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370977759361267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933575548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383617997169495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784933590190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300740718841553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933605101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970896065235138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933619489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276036500930786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933633827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289855420589447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933648483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971150875091553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933662827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248345673084259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933662827, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933662828, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933747220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5046277642250061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933747220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933747221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784933761978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13120490312576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933776634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594689965248108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933791555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133366197347641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933806396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038285076618195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933820814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12821829319000244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933835273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12660683691501617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933849962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289212554693222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933864164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323247253894806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933878649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354707032442093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933893195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12501078844070435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933907433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12858030200004578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933921734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13447900116443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933936223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107462227344513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933951099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291963756084442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784933951100, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784933951100, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934034033, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061578750610352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934034033, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934034034, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784934048792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058283925056458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934063250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290827989578247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934077291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303296267986298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934092017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239861398935318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934106915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961581349372864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934121344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583142518997192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934135900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302293986082077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934150587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218237578868866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934164785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358364075422287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934179512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13729922473430634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934193973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12357193976640701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934208946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321016937494278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934223667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12575945258140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934238196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288527101278305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934238196, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934238196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934323882, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5079554319381714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934323883, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934323883, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784934337993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646940886974335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934352737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13634203374385834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934367141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571444153785706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934381300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994985818862915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934395569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656319677829742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934410027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828469812870026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934424456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12738151848316193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934438807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12628492712974548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934453202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14212629199028015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934467988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14102551341056824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934482722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733325362205505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934497296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856151163578033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934511749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327178180217743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934525703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142827153205872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934525703, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934525703, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934612474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5100165009498596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934612475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934612475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784934627367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14128318428993225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934642493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14238598942756653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934657333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13526351749897003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934672246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955073058605194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934687486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14466935396194458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934702183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14097295701503754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934717353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12588156759738922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934732441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13749440014362335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934747599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170044124126434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934762722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611991703510284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934777769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137721985578537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934792340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329604983329773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934807316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343197524547577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934822334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655684888362885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934822336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934822336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934907121, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5121243000030518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934907121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934907122, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784934921649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13866978883743286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934936514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817159831523895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934951569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193310797214508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934966816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328168660402298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934982109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371635973453522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784934997452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13992708921432495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935012785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275635242462158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935028246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14124181866645813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935043033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1408659815788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935057832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13517534732818604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935072601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12779229879379272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935087236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12992382049560547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935101919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420814275741577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935116776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275714635848999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935116777, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935116777, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935201321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5146987438201904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935201322, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935201322, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784935216042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208593428134918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935230676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368894726037979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935245444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347411155700684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935260045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195888698101044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935274777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392986357212067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935289466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13359524309635162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935304670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393634557724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935319487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132762148976326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935334027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13983596861362457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935349041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194821774959564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935364060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879741191864014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935378908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312377154827118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935393839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247929513454437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935408767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354905664920807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935408767, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935408768, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935492910, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5180987119674683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935492910, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935492910, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784935507916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363142311573029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935522742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303867191076279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935537418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067308068275452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935552201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043242692947388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935566906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351919323205948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935582060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532361388206482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935596666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542282581329346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935611683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12404372543096542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935626334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107341527938843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935640696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114838302135468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935655549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303464025259018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935670202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333281695842743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935685064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084353506565094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935699700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019680976867676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935699701, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935699701, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935784152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5226544737815857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935784153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935784154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784935799072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782382011413574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935813944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324256956577301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935828707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338273763656616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935843321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303848773241043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935857924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712883949279785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935872391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192646205425262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935887346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566800951957703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935902254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328504979610443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935917144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13145823776721954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935932161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914792239665985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935946999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382640480995178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935961365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376328468322754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935976296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319892704486847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935990913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341636955738068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784935990914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784935990914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784936075765, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5288664698600769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784936075766, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784936075766, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784936090513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309298723936081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936105292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484163582324982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936119949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380932331085205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936134616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333598792552948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936149141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272531896829605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936163822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293255090713501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936178491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600795924663544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936193183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402349710464478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936208122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13630622625350952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936223123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720370829105377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936238073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347140222787857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936253592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713760137557983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936268697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087749481201172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936283147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1236271932721138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936283148, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936283148, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936369677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5375683307647705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936369678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936369678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784936384321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323150396347046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936399243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325889378786087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936413992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364316523075104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936428611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792292773723602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936443200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635781407356262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936457847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113835453987122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936472416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272830069065094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936487106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494476675987244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936501640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12469716370105743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936516619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046114146709442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936531370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326279640197754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936546270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13437965512275696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936561225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14056025445461273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936576003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362370252609253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936576004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936576004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936663025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.550406813621521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936663026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936663026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784936678355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666358590126038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936693237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13157066702842712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936707851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326279193162918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936722697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13910675048828125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936738121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013368844985962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936753589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843712210655212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936768753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508574664592743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936783959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712678849697113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936799152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475729525089264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936814505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069666922092438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936829782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1405787318944931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936844926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346164047718048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936859970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129020094871521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936875189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333719938993454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936875189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936875190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936961483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5654758810997009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936961484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936961484, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784936976832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14211831986904144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784936992219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280479609966278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937008070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334669142961502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937023417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067477941513062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937038861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521920144557953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937053992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337593048810959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937069100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426989316940308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937084434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571999967098236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937099666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116572797298431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937115173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328655481338501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937130496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289807736873627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937145851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13696204125881195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937160767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613568246364594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937176240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752590119838715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937176240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937176240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937260981, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5822179913520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937260982, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937260982, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784937276140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316874623298645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937291621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524475693702698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937306827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420315086841583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937322473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319865882396698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937337463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725003600120544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937352407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295015513896942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937367498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361451029777527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937381992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12403292208909988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937396764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087140023708344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937411393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316005289554596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937426133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102664053440094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937440735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315045952796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937455442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889645993709564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937470170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078606128692627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937470171, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937470171, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937556080, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5962616205215454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937556081, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937556081, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784937570525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1223122775554657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937585255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692785263061523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937599836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960711121559143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937614599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132970929145813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937629705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193121552467346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937644828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12166789919137955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937660069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070692121982574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937675118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13206368684768677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937689958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370457410812378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937704425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281294971704483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937719449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333521604537964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937734448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786714941263199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937749426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162790447473526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937764504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12541361153125763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937764505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937764505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937850776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6104215979576111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937850777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937850777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784937865762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12144467234611511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937880533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12756207585334778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937895574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12882624566555023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937910434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13213808834552765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937925334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12153444439172745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937940242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490822911262512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937954971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295929253101349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937969809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12177200615406036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784937984893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12715664505958557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938000160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272268444299698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938015109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11851995438337326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938030567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318323165178299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938045821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717083096504211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938061024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12850505113601685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938061025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938061025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938149001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6239259243011475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938149002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938149002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784938163685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12820583581924438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938178842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11873514950275421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938193908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12177902460098267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938209058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12143519520759583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938223996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12397589534521103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938238900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12616348266601562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938253834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638250201940536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938268916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12266775220632553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938283889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12555143237113953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938298852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12399408966302872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938313568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115031898021698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938328425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398573011159897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938343198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014449596405029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938358228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12482797354459763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938358230, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938358230, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938445140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6372470855712891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938445141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938445141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784938460356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11695849895477295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938475445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497151106595993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938490659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294596642255783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938505991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12181269377470016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938520827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12381323426961899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938535643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678199470043182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938549972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12024708837270737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938565465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1220075860619545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938580988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12060094624757767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938596615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287448555231094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938612228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12081152200698853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938627587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12167034298181534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938642782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12474670261144638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938658110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11736506968736649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938658111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938658111, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938744363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6602479815483093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938744363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938744364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784938759589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11939649283885956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938774598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11208266019821167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938789715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100817546248436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938805017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294769495725632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938820288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849444568157196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938835788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187722325325012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938851431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633173376321793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938867149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039190530776978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938882380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293262451887131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938898004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11980141699314117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938913498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317829042673111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938928932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366777867078781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938944070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928861051797867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938959160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618045717477798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784938959161, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784938959161, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784939044454, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6937454342842102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784939044455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784939044456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784939059311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995279043912888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939074545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940624564886093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939089751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303195148706436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939104842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529191792011261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939119994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408238112926483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939134910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11694838851690292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939150050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11908195912837982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939165178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182609498500824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939180224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087748259305954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939195163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724633723497391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939210186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156880185008049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939225014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577170342206955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939239776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275888979434967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939255061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12450011074542999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939255062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939255062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939339102, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7192524075508118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939339103, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939339103, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784939354423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12364639341831207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939369671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161765530705452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939384663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11577105522155762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939400033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215982735157013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939415203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10941796749830246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939430474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106391996145248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939445135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252942681312561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939460386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784813016653061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939475832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083297431468964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939491735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11616229265928268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939507530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299232184886932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939523213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147681400179863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939539145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126273974776268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939554958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224939674139023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939554959, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939554959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939637442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7523471713066101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939637443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939637443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784939653285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12552717328071594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939668928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391382664442062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939684521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11569991707801819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939700253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286569386720657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939715736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188555508852005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939731436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182413175702095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939747347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12112338095903397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939763283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200919970870018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939779054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501278728246689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939795166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220811307430267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939810963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479891836643219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939826402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11531636118888855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939841689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12500953674316406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939857310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11559991538524628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939857311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939857311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939943149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7597017288208008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939943150, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939943150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784939958565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208360493183136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939974040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12020669132471085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784939989533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308158934116364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940005308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11444103717803955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940020907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079978421330452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940036511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504068225622177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940052217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168321818113327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940067521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067923903465271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940083133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910087823867798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940098528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795875638723373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940114285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209182977676392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940129986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602448135614395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940145775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12089183926582336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940161851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11972643435001373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940161852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940161852, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940246766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7638093829154968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940246767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940246767, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784940262665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185598075389862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940278071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176624596118927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940293374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544173210859299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940309173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451605707406998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940324808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258720815181732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940340341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032528281211853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940355929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11881006509065628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940371386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413559317588806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940386809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10359210520982742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940402254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223335772752762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940417300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062302216887474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940432597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153040826320648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940447870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025074869394302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940462971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479317396879196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940463012, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940463012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940547628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664027810096741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940547629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940547629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784940562953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448537021875381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940578612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680893808603287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940594342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11492560803890228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940609609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014033108949661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940625161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063852310180664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940640904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363869160413742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940656347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286409199237823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940671634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11169115453958511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940686858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657290160655975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940701856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067714542150497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940716771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031159967184067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940731669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349481344223022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940746517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605772584676743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940761512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504526644945145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940761512, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940761513, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940845707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7679443955421448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940845708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940845708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784940860556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599172115325928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940875772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11353108286857605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940890892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09602254629135132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940905973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330890119075775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940921050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242339760065079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940935822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443577170372009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940950970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0881919115781784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940965735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869967937469482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940980857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10153903067111969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784940996152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773420870304108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941011471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696043819189072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941026615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016818344593048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941042002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996507927775383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941057431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751274228096008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941057432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941057432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941141427, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688356637954712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941141428, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941141429, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784941156493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11522859334945679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941171566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119188740849495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941185975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383007675409317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941201389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598888993263245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941216515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989817440509796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941231886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191570967435837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941247171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591349005699158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941262580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703720152378082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941277663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788559377193451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941293012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084107756614685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941308119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487118363380432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941323210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061650738120079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784941338294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820775479078293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed552965065.log b/recommendation_v4/rcp_logs/gbs_8192/seed552965065.log new file mode 100644 index 000000000..730da9890 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed552965065.log @@ -0,0 +1,1815 @@ +:::MLLOG {"namespace": "", "time_ms": 1784658183613, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784658183613, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784658204806, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "seed", "value": 552965065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784658204808, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784658204809, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784658204809, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784658987475, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784658987476, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784658987896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784659258248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799943029880524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784659270346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899414241313934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659283258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874348998069763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659296098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13925470411777496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659309155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904842734336853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659322331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886089622974396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659335671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13916324079036713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659348835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13850216567516327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659362308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880807161331177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659375682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887231051921844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659389295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13896244764328003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659402796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890060782432556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784659416637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13893844187259674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659430225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825617730617523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659430226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659430226, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659520036, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4999590218067169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659520036, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659520037, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784659532801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389658898115158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659545782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386823207139969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659559888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387050598859787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659573336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388169527053833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659586970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869225978851318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659600497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882645964622498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659613630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860014081001282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659626950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858339190483093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659641444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884922862052917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659655206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862979412078857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659669222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867130875587463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659683073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387033760547638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659696684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851049542427063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659710940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138844832777977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659710941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659710941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659795829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001230835914612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659795830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659795830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784659809498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832128047943115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659823389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853776454925537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659837269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381119191646576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659851142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785172998905182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659864534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860253989696503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659877977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831062614917755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659891469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799473643302917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659905539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796545565128326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659919455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13742361962795258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659933204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774333894252777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659947184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799074292182922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659960893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13808418810367584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659975178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804316520690918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659988804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378697156906128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784659988805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784659988805, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660071834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003073811531067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660071835, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660071835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784660085173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737033307552338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660098871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377810686826706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660112434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718874752521515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660126846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748934864997864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660140823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768510520458221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660155088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378512680530548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660169229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371222287416458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660183201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757918775081635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660197628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710801303386688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660211696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385217159986496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660225600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662850856781006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660239458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378578245639801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660253390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13573578000068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660267484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681861758232117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660267485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660267485, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660351246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005672574043274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660351247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660351247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784660365102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13639608025550842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660379095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577444851398468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660392820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758306205272675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660406832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496892154216766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660420885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367558091878891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660434701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387584686279297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660448875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574974238872528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660462568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562487065792084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660476538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378091275691986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660490404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540299236774445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660504248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14615002274513245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660518486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577800989151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660532288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418659567832947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660546388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771024346351624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660546388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660546389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660631430, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500906229019165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660631431, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660631431, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784660645333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354517787694931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660659161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13962006568908691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660673197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615639507770538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660687115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352316617965698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660701297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349768340587616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660715522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919053971767426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660729472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335921436548233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660743421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306336104869843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660757001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324172794818878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660770427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13164928555488586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660784307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124771416187286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660797816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508455455303192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660811457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178212940692902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660825446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874736428260803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660825446, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660825447, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660908517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014082789421082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660908518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660908518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784660922304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377967476844788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660936008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341419398784637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660950294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358257234096527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660964390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351969838142395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660978636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791343569755554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784660992931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800505757331848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661006869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549545407295227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661021037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293003261089325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661035366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334085762500763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661049860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327323168516159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661064046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373106718063354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661078338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13421498239040375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661092656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135316401720047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661106573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355106115341187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661106574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661106574, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661190483, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022820234298706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661190484, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661190485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784661204410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253094255924225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661218260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377249240875244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661232126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13589408993721008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661246353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355898231267929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661260303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396108865737915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661274059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333117038011551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661289297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135166734457016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661303728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346953690052032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661317941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289061188697815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661332386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318179965019226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661346316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579308986663818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661360705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335703432559967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661374850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355956971645355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661388850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059742748737335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661388851, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661388851, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661473112, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5034599304199219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661473113, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661473114, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784661487881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806092739105225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661502046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500580191612244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661516315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354016661643982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661530618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118544220924377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661544661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364659249782562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661558829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343492567539215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661573216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695314526557922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661587531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398835062980652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784661601713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13299560546875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661615731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294887512922287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661629374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276707947254181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661643181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13311472535133362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661657054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12972883880138397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661670665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12462257593870163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661670666, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661670666, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661753894, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5047202706336975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661753895, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661753895, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784661767886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139000535011292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661781651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588716089725494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661795725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342653214931488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661809862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063772022724152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661823671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1281563639640808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661837921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12640880048274994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661852186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12905199825763702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661866091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243651390075684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661880066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554884493350983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661894007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1250339150428772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661907687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12846575677394867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661921467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435883820056915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661935552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112753629684448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661949787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920919060707092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661949787, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784661949788, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662032866, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5063016414642334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662032867, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662032867, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784662046925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044272363185883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662060755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12919020652770996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662074231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331424117088318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662088826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12385617196559906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662103265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296423375606537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662117207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12588661909103394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662131187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12999510765075684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662145466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12189626693725586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662159255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576778769493103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662173588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13706083595752716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662187565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12362799048423767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662202067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330584973096848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662216329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12564173340797424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662230570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290406435728073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662230571, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662230571, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662313135, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5081444978713989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662313136, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662313136, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784662326956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665038347244263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662341380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669784367084503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662355162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560304045677185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662368815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14022551476955414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662382635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665997982025146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662396513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384442001581192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662410378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12757378816604614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662424138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12620095908641815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662437880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14237329363822937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662452133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14098887145519257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662466258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273718625307083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662480203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283930540084839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662494110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242369890213013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662507508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13152214884757996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662507509, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662507509, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662590272, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5102068781852722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662590273, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662590273, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784662605376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14137548208236694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662620196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14257048070430756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662634523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354004442691803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662649107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13933195173740387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662663999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1450531780719757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662678298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14101271331310272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662692804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12612766027450562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662707299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376419961452484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662722072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317634880542755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662736663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615137338638306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662751222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782374560832977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662765428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331111192703247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662780248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454820215702057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662794592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13660554587841034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662794592, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662794593, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662875526, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5123598575592041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662875527, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662875527, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784662889528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386594921350479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662903995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380886286497116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662918550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316414326429367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662932981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329125612974167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662947436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337766796350479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662962411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13967552781105042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662977386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275635242462158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662992348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14123030006885529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663006738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1406668871641159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663021574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508111238479614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663036075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12762971222400665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663050261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018545508384705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663064420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411881029605865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663078902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777194380760193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663078903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663078903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663163329, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5149442553520203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663163329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663163330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784663177552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322093904018402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663191911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13671553134918213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663206410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343705236911774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663220893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13186843693256378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663235501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394153118133545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663249896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334502398967743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663264527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383246958255768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663278962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258393108844757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663293173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13997139036655426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663308131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166727125644684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663322573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128742977976799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663336884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121894001960754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663351262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212114572525024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663365726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357986032962799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663365726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663365727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663449410, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5184434056282043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663449411, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663449412, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784663464036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335851103067398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663478674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296156585216522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663493285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13054616749286652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663507977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038809597492218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663522430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508571684360504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663537527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533343374729156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663552010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354304701089859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663566769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398926913738251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663581248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104626536369324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663595371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104477524757385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663610112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303461343050003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663624308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304094970226288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663638620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107267022132874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663652730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13021479547023773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663652731, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663652731, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663735697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5232583284378052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663735698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663735698, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784663750116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779330253601074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663764544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323932260274887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663778776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334737241268158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663793103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027417659759521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663807236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719541788101196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663821373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322733610868454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663835884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575130701065063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663850269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329377144575119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663864578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119536638259888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663879099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887722790241241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663893624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337345391511917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663907539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777250051498413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663922370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317223608493805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663936529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325555622577667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663936529, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784663936530, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664019013, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5301088690757751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664019015, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664019015, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784664033213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112807273864746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664047409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350865364074707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664061744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762417435646057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664075991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333334892988205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664090050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12729424238204956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664104189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12927928566932678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664118383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577621638774872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664132264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12398124486207962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664146615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358814239501953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664161056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717389106750488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664175364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455830514431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664190200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12695352733135223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664204563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305464506149292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664218451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12357718497514725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664218451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664218452, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664301268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5401998162269592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664301269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664301269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784664315804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247530162334442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664330167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324150264263153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664344409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364361226558685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664358640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781920075416565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664372969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622289896011353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664387365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13128021359443665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664401624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271827667951584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664415897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350192427635193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664430015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12476976960897446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664444586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038043677806854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664458965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321927160024643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664473558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388708233833313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664488355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1404450535774231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664502700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603733479976654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664502701, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664502702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583624, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5559850335121155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583625, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664583625, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784664598485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650840520858765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664613074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13111497461795807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664627362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239818811416626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664641811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881073892116547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664656898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12973974645137787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664672060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382676213979721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664686580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345081925392151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664701364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686566054821014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664715997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344965249300003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664730830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130527526140213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664745595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399218738079071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664760337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13455846905708313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664775010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12854449450969696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664789772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333451122045517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664789773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664789774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664871946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5728837847709656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664871947, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664871947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784664886813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1413743495941162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664901339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322048008441925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664916320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313816487789154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664930942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019835948944092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664945959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13497790694236755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664960862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318601250648499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664975595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335376799106598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664990276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515476882457733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665004863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077928125858307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665019748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323266625404358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665034568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241222500801086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665049435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669615983963013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665063909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358635276556015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13683247566223145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665078735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665078735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163048, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5888171792030334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163048, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665163049, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784665177608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101112842559814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665192469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13461506366729736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665207144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337691128253937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665222333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13159269094467163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665236903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649587333202362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665251331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881624698638916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665266414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308638334274292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665280753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12234139442443848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665295309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12991908192634583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665309767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308651715517044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665324290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019444048404694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665338480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046270608901978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665352764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12806424498558044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665367265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959638237953186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665367266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665367266, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665452271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6012089848518372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665452272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665452272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784665466355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12115747481584549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665480739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12562403082847595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665495090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275976449251175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665509580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216876983642578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665524300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072320818901062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665538826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1206408143043518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665553626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916713953018188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665568394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130523681640625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665582988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357566863298416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665597193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12666645646095276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665612212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13168780505657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665626413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652933806180954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665640704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11997154355049133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665655066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366432696580887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665655067, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665655067, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665739123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6119140982627869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665739124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665739124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784665753246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12019892036914825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665767250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12546274065971375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665781473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274472028017044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665795657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040021061897278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665809782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969975382089615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665824037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133904367685318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665838279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760086357593536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665852420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12017550319433212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665866857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12554991245269775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665881454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12509562075138092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665895741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682901531457901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665910541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13018769025802612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665925066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126125305891037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665939472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12757688760757446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665939472, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784665939473, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666025890, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.620954692363739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666025891, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666025891, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784666039855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254407912492752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666054821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168498620390892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666069581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058831006288528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666084400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11953532695770264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666098945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12219134718179703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666113529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12452683597803116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666127980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447031795978546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666142621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158572673797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666157203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12373241782188416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666171695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12193397432565689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666186277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139625757932663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666200716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12083431333303452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666215042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738841444253922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666229485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303372472524643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666229485, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666229486, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666316188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6297642588615417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666316189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666316189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784666330772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469430476427078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666345297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11310458183288574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666359914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12785103917121887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666374952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11945724487304688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666389483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12211843580007553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666403922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11416764557361603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666418097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758730560541153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666432930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1204940676689148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666447345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11945618689060211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666461750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1114073395729065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666476097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11857143044471741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666490393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214645653963089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666504684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12401822209358215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666519120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638699471950531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666519120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666519121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666605607, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6432831287384033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666605608, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666605609, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784666620001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11754223704338074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666634170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094605922698975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666648244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880561172962189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666662278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260897666215897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666676383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790131241083145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666690591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12273216247558594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666705032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114004947245121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666719719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12025664746761322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666733994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052774101495743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666748680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203790009021759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666763411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204218119382858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666778029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177803575992584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666792469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890824347734451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666806903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11686602979898453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666806904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666806904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666891418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6628134250640869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666891419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666891419, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784666905895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12059533596038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666920521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12058954685926437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666935231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12281739711761475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666949832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503115504980087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666964421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346498131752014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666978860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11680986732244492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666993588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969521641731262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667008320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896408349275589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667022873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883548855781555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667037300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11796131730079651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667051666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624391376972198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667065856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11689098924398422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667080234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355000734329224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667094980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568703293800354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667094981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667094981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667179442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6876556873321533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667179443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667179443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784667194176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12357061356306076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667208902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11668358743190765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667223556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678680032491684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667238800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12324497103691101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667253694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042983829975128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667268472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192025244235992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667282693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352083832025528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667297709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912134498357773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667312473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115780472755432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667327682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679071933031082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667342604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12447724491357803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667357352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537674814462662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667372347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278441995382309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667387195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262689530849457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667387195, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667387196, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667470654, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7091096043586731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667470655, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667470655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784667485581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713424861431122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667500225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452675610780716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667514931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11777949333190918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667530024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11344048380851746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667544675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951036751270294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667559646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11982087045907974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667574951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12213747203350067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667590141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12122014909982681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667605163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11611546576023102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667620671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380874365568161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667635710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679720133543015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667650493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173253282904625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667665244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12641097605228424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667680669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634532362222672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667680670, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667680670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667764929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7291173934936523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667764930, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667764930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784667780167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12371691316366196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667795656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12096355110406876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667810770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430855840444565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667825922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11629936099052429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667840844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955443978309631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667855876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597130447626114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667870868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11884675920009613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667885693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268441379070282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667900699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992365330457687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667915594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943824797868729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667930866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12353833764791489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667946250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10787832736968994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667961292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12156340479850769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667976817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976757645606995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667976818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784667976818, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668060889, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7434591054916382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668060890, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668060890, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784668076049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12144766747951508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668090944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1202075183391571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668105896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115965835750103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668121261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11726990342140198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668136286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428761690855026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668151340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596517473459244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668166223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063826620578766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668181134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560311168432236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668195992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565987229347229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668210978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434678196907043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668225668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083378866314888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668240593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311125010251999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668255469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303956061601639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668270327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163102388381958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668270373, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668270374, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668357095, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7527211308479309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668357097, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668357097, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784668371902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049090176820755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668387103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804679989814758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668402423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168646588921547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668417330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196686327457428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668432520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903113335371017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668447894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584555357694626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668463010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11360613256692886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668477803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425746977329254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668492919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.117919921875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668507816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229243874549866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668522530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244238376617432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668537276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459369957447052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668551786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855696350336075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802388936281204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566430, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668566430, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668652507, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7561548352241516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668652508, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668652508, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784668667121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776841640472412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668682032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459986865520477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668696847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09808070212602615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668711460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104339100420475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668726224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396808922290802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668740817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058289185166359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668755434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08985555917024612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668769970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000413447618484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668784767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409477353096008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668799626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11719410866498947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668814712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837613791227341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668829616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105680674314499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668844917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145687311887741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668860082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083582192659378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668860083, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668860083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668944046, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7575886249542236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668944046, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668944047, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784668958922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11725829541683197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668973782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249864846467972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668988127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458160191774368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669003529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071750670671463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669018157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041751712560654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669033115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448572784662247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669047902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781173408031464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669062954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943431407213211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669077824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09833849966526031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669092917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944432020187378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669107763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054118201136589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669122685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837013274431229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669137551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842226445674896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669152464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131020113825798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669152464, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669152465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669234337, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7582312226295471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669234338, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669234338, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784669249250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125095933675766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669263587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10024839639663696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669278317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10399632155895233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669293276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748755186796188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669308188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136133968830109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669323151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140298843383789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669338083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084249079227448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669353126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602894425392151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669368150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773903876543045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669382904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656298696994781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669397719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305981874465942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669411948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09556512534618378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669427331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09842762351036072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669442093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843703895807266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669442094, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669442095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669523565, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7607043981552124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669523566, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669523566, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784669537901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0926835909485817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669552569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499082505702972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669567108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043862253427505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669581840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09891252964735031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669596354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110629215836525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669611000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156409978866577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669625606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09795581549406052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669640038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156366765499115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669654720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636813193559647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669669213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547517985105515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669683907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938441008329391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669698093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055293157696724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669712415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09825889766216278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669727074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733496397733688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669727074, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669727075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669808692, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7623792290687561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669808693, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669808694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784669823746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510244220495224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669838801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536855459213257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669853494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598364472389221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669868376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419239848852158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669883241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989052876830101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669898024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094214230775833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669913102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651808977127075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669927407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09615535289049149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669942349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823020339012146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669956926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890167206525803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669971303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711908131837845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669985820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546462774276733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670000129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623266339302063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670014535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567256599664688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670014535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670014536, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670096955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7642495632171631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670096956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670096956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784670111148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162676870822906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670125576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11619577556848526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670139987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082889661192894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670154763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10267677158117294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670169285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0912107601761818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670183915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837912023067474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670198363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045994758605957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670212983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778253525495529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670227381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833050310611725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670241906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034714639186859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670256893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132354587316513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670272011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09982895851135254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670287188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005252599716187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670302031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303727328777313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670302032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670302033, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7652899026870728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670385091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784670400129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081085056066513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670415040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190222412347794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670429680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055015027523041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670444492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09185964614152908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670458768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11512662470340729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670473104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722924768924713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670488726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155542731285095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670503727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109859496355057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670518943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246204167604446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670533887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555077344179153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670548947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207500845193863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670564202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1233496367931366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670579117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988876432180405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670593996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175902932882309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670593997, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670593998, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670677499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659822106361389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670677500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670677500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784670692403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041853278875351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670707229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1188519150018692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670721978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972927510738373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670737113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551214963197708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670751924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09680968523025513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670766840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308252066373825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670782117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640640556812286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670797243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679514706134796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670812294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681105941534042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670827043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955725818872452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670842260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470730394124985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670857373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775639861822128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670872459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11724931001663208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670887416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818365961313248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670887416, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670887417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670972195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7664766311645508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670972196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670972196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784670987319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430183798074722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671002488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581480503082275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671017659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129315197467804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671032759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11803755164146423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671047761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463473737239838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671062935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11893336474895477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671078378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12742270529270172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671093433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12101880460977554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671108491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610987037420273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671123977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090923845767975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671139177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10860344022512436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671154471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804637521505356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671169621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151478961110115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671184928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478086560964584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671184929, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671184929, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671269494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768110990524292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671269495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671269495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784671284903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12000101059675217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671300290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728660970926285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671315381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903028398752213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671331009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800377279520035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671346364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121080070734024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671361509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979808866977692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671376581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363855749368668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671391490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803906619548798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671406323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671205073595047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671420829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081226378679276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671435893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422738641500473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671450616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738171845674515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671465463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440685600042343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671480272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075122117996216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671480272, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671480272, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671563871, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692722678184509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671563872, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671563872, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784671578557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972961038351059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671593438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126078873872757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671608293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970630869269371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866563767194748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671638196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503998935222626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671653027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500345379114151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671667828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482362657785416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671682712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499686747789383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671697500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202785581350327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671712502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200610026717186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671727533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164766177535057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671742604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167405754327774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671757880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052410513162613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671772644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12051520496606827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671772646, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671772646, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671857530, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696687579154968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671857531, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671857531, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784671872683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068937182426453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671887679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09660215675830841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671902652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506575554609299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671917711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353858768939972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671932395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871294885873795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671947337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516069084405899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671962072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690950602293015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671976661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220815777778625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671991603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489332675933838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672006488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793318599462509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672021111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053493320941925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672035775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476066708564758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672050160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678164660930634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672064753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251594334840775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672064754, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672064754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672149188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702942490577698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672149189, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672149189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784672163786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976334869861603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672178541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09443977475166321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672193090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09640350937843323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672207751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09660836309194565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672222396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09681497514247894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672237299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09481257945299149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672251980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057759284973145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672266708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689074009656906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672281779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284414142370224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672296635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890453308820724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672311729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473132878541946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672326660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343988984823227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672341540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11470083892345428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166082322597504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672356955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672438824, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708221077919006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672438825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672438825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784672453919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09870336204767227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672468977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716582834720612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672484296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977748036384583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672499121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111355036497116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672514110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10075461119413376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672528958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774217545986176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672543646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068297922611237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672558408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626283288002014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672573429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247038304805756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672588123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176991671323776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672602805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983641654253006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672617423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291752964258194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672632122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973136872053146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672646733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457201838493347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672646735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672646735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672727401, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7709139585494995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672727402, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672727402, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784672742045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017722636461258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672756595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071045771241188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672771287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09126114100217819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672785923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313673317432404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672800735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067257672548294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672815167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808424651622772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672829638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991548001766205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672844333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313700139522552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672859211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033799707889557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672874171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416644811630249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672889222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264048725366592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672903938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823312401771545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672918987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050790399312973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672933992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093808263540268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672933992, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784672933993, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673015791, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7714128494262695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673015792, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673015792, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784673030514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10372438281774521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673045230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383300483226776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673059743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039569079875946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673073966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024923324584961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673089412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730501264333725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673104099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125790536403656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673118850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889412671327591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673133853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221643000841141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673148630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163705587387085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673163369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510937124490738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673178090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386615991592407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673192729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10281877964735031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673207613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434514284133911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673222328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10681722313165665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673222329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673222329, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673307266, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719566226005554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673307267, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673307267, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784673322207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842598229646683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673337138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217338055372238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673351896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413184016942978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673366832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138615220785141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673381881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503864288330078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673396687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246854811906815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673411918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163788497447968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673427083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11032926291227341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673442617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377698928117752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673458216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611158609390259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673473494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358814150094986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673488901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048101857304573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673504637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495594143867493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673520107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09945116192102432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673520107, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673520108, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673605239, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724649310112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673605240, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673605240, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784673620687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258351266384125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673635907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09946665912866592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673651186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635492950677872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673665950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072574764490128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673681333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799048840999603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673696120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942861437797546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673711122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755161195993423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673726207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526735872030258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673741177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421159654855728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673756101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090754181146622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673770885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103833869099617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673785604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311432927846909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673800301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531531274318695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673814811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762470960617065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673814812, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673814812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673898097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723333835601807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673898099, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673898099, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784673913002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236167907714844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673927860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068614780902863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673942963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920953750610352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673957896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494718700647354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673972692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09432478249073029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673988026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214663833379745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674002904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120684653520584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674017763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076372042298317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674032874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103740394115448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674048123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340366512537003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674063100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963961482048035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674078319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463237553834915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674093397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606727749109268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674108620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310696065425873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674108620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674108621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674192029, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772475004196167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674192030, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674192030, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784674207354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11335679888725281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674222492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921350866556168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674237551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990176349878311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674252858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751769691705704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674267666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10136682540178299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674282227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810829699039459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674298093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11782412230968475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674313669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903023928403854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674329121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648276448249817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674344920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12224160879850388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674360411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737156867980957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674375932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086719483137131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674391564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468353122472763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674406973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974818468093872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674406974, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674406974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674490114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725011706352234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674490115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674490115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784674505555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369781941175461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674521171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095953956246376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674536619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111204326152802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674552136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066138744354248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674567621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045481488108635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674583124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340418457984924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674598636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667996108531952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674614224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485638469457626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674630044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223098635673523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674645795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12006304413080215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674661752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060485988855362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674677521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758163571357727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674693173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743365436792374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674708868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11945103108882904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674708869, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674708869, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674791004, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728175520896912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674791005, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674791006, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784674806601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11982090771198273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674822000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802178829908371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674837140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970112472772598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674852242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422348022460938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674867537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745739191770554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674883117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521004885435104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674898403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198542267084122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674913821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656239628791809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674929299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290138959884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674944486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10586400330066681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674959635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376823484897614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674974841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608775913715363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674989934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699650436639786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675005327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553760826587677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675005328, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675005328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675087926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733796834945679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675087926, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675087927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784675103023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985536128282547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675118360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11963549256324768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675133406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065738648176193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675148524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370307952165604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675163689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601575464010239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675178810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934940725564957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675194046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823490470647812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675209527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005768924951553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675225075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124274879693985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675240255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278194189071655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675255654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992828756570816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675271142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039676517248154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675286588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148940026760101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675301907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090904027223587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675301907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675301908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675386649, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734882831573486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675386650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675386650, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784675401587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816207319498062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675416920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108239740133286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675431795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082352697849274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675446864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315161198377609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675461597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590894520282745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675476593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198033392429352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675491428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081482470035553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675506086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772236436605453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675520732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972389280796051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675535382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018366739153862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675550101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551013797521591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675564622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1045004203915596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675579481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025072634220123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675594480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11628006398677826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675594481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675594482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675679527, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736825942993164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675679528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675679529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784675694222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461648344993591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675708909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559825599193573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675723729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083957016468048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675738870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09228663146495819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675753594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363030225038528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675768074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408120602369308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675782819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255050659179688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675797883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254727095365524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675812814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721845924854279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675827820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362335294485092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675842744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958860069513321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675857965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101820319890976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675873303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772555321455002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675888398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714070498943329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675888399, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675888399, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675970186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737115621566772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675970187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675970187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784675985049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10972150415182114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676000214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336288064718246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676015201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057031899690628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676029996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952414035797119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676045610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11580689251422882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676060773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035864800214767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676075785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362912714481354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676091068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792698711156845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676106235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840388387441635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676120993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087806299328804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676135958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105596125125885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676150782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0992150530219078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676165605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701523721218109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676180413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08735102415084839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676180414, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676180414, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676262471, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738859057426453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676262472, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676262473, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784676277537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09522689133882523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676292590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109552532434464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676307686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756984353065491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676322759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154039412736893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676337763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206974297761917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676353003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0902676060795784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676367973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362037271261215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676382986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09716498851776123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676398031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039154902100563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676413204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638890415430069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676428456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056242138147354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676443509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819084197282791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676458528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648135095834732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676473732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317320376634598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676473732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676473733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676557917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773803174495697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676557918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676557919, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784676573088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09651698917150497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676588604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531379282474518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676604078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09497105330228806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676619434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249438881874084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676634842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979381799697876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676649867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048148036003113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676664798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922686755657196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676680344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090120881795883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676695000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900279134511948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676709851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977187752723694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676725045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658758133649826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676740170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454875230789185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676755101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10964180529117584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676770074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920406132936478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676770075, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676770075, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676852533, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742225527763367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676852534, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676852534, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784676867400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668321698904037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676882238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047973558306694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676897027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382308810949326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676911880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999973937869072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676926853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667775571346283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676941777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09605084359645844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676957300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10548501461744308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676972207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338583588600159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676986980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11048562079668045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677001847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09917221963405609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677016488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09752672165632248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677031284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231642425060272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677046068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09088427573442459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677060955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739145427942276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677060956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677060956, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677142562, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742472290992737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677142563, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677142563, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784677157194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09944417327642441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677172155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115378350019455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677186931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09906347095966339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677202053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000346392393112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677217029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10043911635875702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677232178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243477672338486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677246999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10215522348880768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677261689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530959069728851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677276641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203330218791962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677291198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296837031841278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677306479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10116874426603317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677321998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823141783475876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677337392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547935962677002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677353079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060882955789566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677353080, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677353081, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677434484, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743372321128845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677434485, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677434485, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784677450101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065770611166954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677465795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834153950214386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677481228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690996795892715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677496958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739587992429733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677512642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273580253124237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677528032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731975734233856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677543601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769302606582642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677559301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985049307346344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677574834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051524356007576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677590336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466697812080383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677606009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313636064529419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677621753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100674718618393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677637505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679648816585541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677653055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012850910425186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677653056, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677653057, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677735412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745670676231384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677735413, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677735413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784677750913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341231524944305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677766285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544803738594055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677781810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09456998854875565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677797645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09875989705324173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677813118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208892822265625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677828899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032607614994049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677844601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692936927080154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677860430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973890125751495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677876073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853261500597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677891688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033433139324188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677907431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270387679338455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677922625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0986272320151329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677938123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124630942940712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677953143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110997274518013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677953143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784677953144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678038194, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747896909713745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678038195, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678038196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784678053043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585133731365204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678068577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483498871326447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678083286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647380352020264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678098220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115065217018127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678113511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632440447807312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678128682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10900268703699112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678143345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783589631319046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678158281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977359265089035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678173183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699149966239929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678187812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10376715660095215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678202744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082145944237709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678217273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10361424833536148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678232370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634340345859528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678247296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09585977345705032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678247344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678247344, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678329785, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744455337524414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678329786, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678329786, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784678344626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09921760112047195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678359395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545545071363449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678374355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258479416370392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678389348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407175868749619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678404339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782988369464874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678419207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234048455953598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678434332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802687704563141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678449633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885881841182709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678464934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11261860281229019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678480151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222001910209656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678495494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11461423337459564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678510811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.099125936627388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678526272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664304345846176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678541553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047402024269104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678541601, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678541601, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678624448, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749985456466675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678624449, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678624450, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784678639697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09798288345336914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678654898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812422096729279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678669940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301998049020767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678685186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11781749874353409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678700101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432977020740509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678714701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105909943580627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678730468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679641366004944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678746023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030281126499176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678761574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520271956920624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678777406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794291645288467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678793261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11125292629003525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678808852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062360554933548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678824426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1130780577659607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678839953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141259968280792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678839953, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678839954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678924234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752209901809692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678924235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678924235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784678939900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720156133174896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678955608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055469512939453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678971471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452797800302505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784678987124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999633371829987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679002704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821057111024857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679018516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828284174203873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679034157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174847394227982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679049727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298618465662003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679065307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004699558019638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679081034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954868048429489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679096551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886732488870621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679112500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453587770462036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679128448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349419713020325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679144283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870437324047089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679144284, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679144284, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679229642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751529812812805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679229642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679229643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784679245329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429392546415329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679260858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12012463808059692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679276413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408296227455139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679292134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11227956414222717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679307911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120575666427612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679323355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1203671544790268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679339040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117851585149765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679354222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903777927160263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679370023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11065398901700974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679385483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889756143093109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679400742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035359650850296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679416536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935057699680328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679432245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431171953678131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679447465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110275238752365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679447466, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679447466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679531755, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751463651657104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679531755, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679531756, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784679546883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320057511329651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679562289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287248134613037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679577761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823465138673782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679593159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123003363609314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679608393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308407783508301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679623823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213038861751556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679639114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454028636217117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679654684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316046863794327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679670041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698448121547699, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679685450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084374263882637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679701323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970314919948578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679716794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133008524775505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679732693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337722837924957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679748640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652883887290955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679748641, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679748641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679831989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754664421081543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679831989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679831990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784679847237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554106742143631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679862768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126850545406342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679878432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12231703847646713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679893983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698774456977844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679909800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962364822626114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679925495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747577250003815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679941001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171978712081909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679956070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260786652565002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679971607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342445969581604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784679986599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184222623705864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680001707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084275171160698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680017008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11136496812105179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680032387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09249947220087051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680047405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10379166901111603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680047406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680047406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680130800, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755693197250366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680130801, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680130801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784680146077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10179179906845093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680161154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0969187542796135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680176094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11491759121417999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680190972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774854570627213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680205708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944300144910812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680220858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445953160524368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680235865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786265879869461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680250932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11337218433618546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680265946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965506196022034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680281055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09585850685834885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680296068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09711847454309464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680310965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09679725766181946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680325795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243436694145203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680340774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008734181523323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680340775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680340775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680424550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755444049835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680424551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680424551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784680439477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10375186800956726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680454735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031138151884079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680469803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438747704029083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680484923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672374814748764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680500043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025138795375824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680515396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990967810153961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680530794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955553501844406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680545930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505404323339462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680561139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067034900188446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680576158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063026413321495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680591658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431532561779022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680606668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082649901509285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680621565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609164088964462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680637201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336471140384674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680637239, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680637240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680721083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757243514060974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680721084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680721084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784680735940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813214629888535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680750968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11209828406572342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680766138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728926956653595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680781469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10773249715566635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680796421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10058476030826569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680811587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10205656290054321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680826782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09628736972808838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680841637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080436035990715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680856396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940848499536514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680871247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275542736053467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680886464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198643058538437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680901373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316538065671921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680916208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998275339603424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680931276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382518917322159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784680931323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784680931323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681015712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757202982902527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681015714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681015714, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784681030785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09797004610300064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681045943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757656395435333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681060853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09886974096298218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681075801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286851972341537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681090508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.094500832259655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681105448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660538822412491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681120590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397319495677948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681135566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040806844830513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681150920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051468625664711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681166001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158427596092224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681181320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09640610963106155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681196747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144086718559265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681211964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225848644971848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681226967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018434688448906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681227015, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681227015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681313851, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7760013937950134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681313852, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681313852, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784681328612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805140644311905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681343702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714544355869293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681358867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085044801235199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681373534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652562230825424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681388656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078267753124237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681403644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904742777347565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681418719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000368669629097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681433848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902563691139221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681448952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921299457550049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681464235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948925465345383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681479503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042422264814377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681494847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09451229870319366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681509894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09560400247573853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681524953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597772151231766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681524954, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681524954, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681608550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762555480003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681608551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681608551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784681623790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305137187242508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681638979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378656327724457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681654178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162705183029175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681669489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593324899673462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681684669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052487045526505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681699773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065224021673203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681715016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10977331548929214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681730107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151757299900055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681745279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764924436807632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681760398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144416242837906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681775518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777242690324783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681791023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153887212276459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681806554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11908216774463654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681821869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942085087299347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681821870, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681821870, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681904552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765254378318787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681904552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681904553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784681919916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610374599695206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681935065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081896185874939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681950681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983973205089569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681966115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080035030841827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681981360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555675625801086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784681996609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09896174818277359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682011832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048213601112366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682027121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615680366754532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682041972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567493736743927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682056686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09437938034534454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682072372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453102737665176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682087370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177437961101532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682102476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09331221878528595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682117699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10050517320632935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682117700, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682117700, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682200153, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765637040138245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682200154, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682200154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784682215388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073098331689835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682230630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328841000795364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682245624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006229817867279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682260765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701052844524384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682275916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350238531827927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682290941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393205285072327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682306132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643699020147324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682321190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870818048715591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682336487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823095053434372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682351534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09733237326145172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682366698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196649819612503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682381539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09786434471607208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682396931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187935084104538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682411874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304316133260727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682411875, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682411875, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682496524, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768917083740234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682496524, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682496525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784682511681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10175113379955292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682526658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848377645015717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682541716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228736490011215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682557178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09397979080677032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682572660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430128127336502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682588078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764440149068832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682603529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618561297655106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682618768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104060173034668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682634224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10555383563041687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682649890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004109680652618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682665532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09755495190620422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682680909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849206894636154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682696345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09876315295696259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682711574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654675215482712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682711575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682711576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682795647, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771489024162292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682795648, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682795649, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784682811150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029288172721863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682826104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10454930365085602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682840810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09227795153856277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682856382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004406213760376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682871424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10887124389410019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682886542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108269914984703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682902140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09618444740772247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682917590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412478446960449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682932738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058831512928009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682947881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997168928384781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682962931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440943390130997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682977938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532116144895554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784682992800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104750782251358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683008017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407165974378586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683008017, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683008018, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683089920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773520946502686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683089921, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683089921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784683104745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936039686203003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683119893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452757775783539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683134770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415691882371902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683149854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088084951043129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683164744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221856832504272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683180135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09768795967102051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683195335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269258916378021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683210358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11251284182071686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683225468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123918950557709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683240686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09529995918273926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683256153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050400510430336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683271731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11248779296875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683287455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162501037120819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683302936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819541454315186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683302937, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683302937, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683384677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776286602020264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683384677, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683384678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784683400362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115775465965271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683415745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717968851327896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683431327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558876395225525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683447148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213841289281845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683463000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812715232372284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683478705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09662627428770065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683494536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557949542999268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683509975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139968991279602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683525963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10141575336456299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683541656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042453944683075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683556761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615433752536774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683572713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991960018873215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683588081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09942445904016495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683603549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498574167490005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683603550, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683603550, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683687466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775029540061951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683687467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683687467, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784683702982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254791915416718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683718626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1113620176911354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683734115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688957571983337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683749700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12101852148771286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683765189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11606157571077347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683780726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202674567699432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683796363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11864793300628662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683811671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450738459825516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683827108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056765615940094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683842562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784120857715607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683858254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796665400266647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683873691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654494911432266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683889309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739057511091232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683904552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550680547952652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784683904552, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683904553, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683989476, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783324122428894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683989476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784683989476, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784684005008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11252786964178085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684020262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431145876646042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684035655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339304804801941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684051243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11678243428468704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684066853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835223644971848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684082180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065356582403183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684097835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536456108093262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684113358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402176320552826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684129273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.118105448782444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684144854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613447427749634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784684160354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11786451190710068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684175855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286060512065887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684191093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797473788261414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684206836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11533725261688232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684206837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684206837, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684292913, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784428596496582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684292917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684292917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784684308237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092977434396744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684323955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071209609508514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684339848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11596709489822388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684367122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018984317779541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684383283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09974063187837601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684411423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825282335281372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684427173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818384796380997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684445650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789960622787476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684461466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10110635310411453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684477211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515380650758743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684492622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442946642637253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684508140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075793206691742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684523514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172831580042839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684539333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300824582576752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684539334, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684539337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684624278, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788059711456299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684624285, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684624286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784684640010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268700659275055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684656090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10788708180189133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684671577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119088530540466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684691140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11675215512514114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684706895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269690841436386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684722661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816689372062683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684738488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09899529069662094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684754172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830911248922348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684770027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874839872121811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684785740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089104562997818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684801566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923922598361969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684817489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282525956630707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684833026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475925356149673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684848913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11158151924610138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684848914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684848914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684935090, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790535092353821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684935091, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684935091, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784684950571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546159744262695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684966373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102665960788727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684981738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11949467658996582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784684997655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1064491793513298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685012756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554107278585434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685028011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192958265542984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685044109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636454820632935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685059357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100832149386406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685074650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322292685508728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685089821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09375985711812973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685105082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11715856194496155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685120343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084690898656845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685135173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845791757106781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685150097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868494212627411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685150098, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685150098, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685235946, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789412140846252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685235947, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685235947, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784685251012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250106453895569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685266319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09355605393648148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685281701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09633991122245789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685297168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830835998058319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685312440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888399183750153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685327700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292658001184464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685342984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09563206136226654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685358303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668633878231049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685373617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588331520557404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685388764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126109212636948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685403996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855853021144867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685419605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964843839406967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685435050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074801534414291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685450263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09885291755199432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685450264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685450264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685537591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792759537696838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685537592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685537592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784685553108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593144595623016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685568462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09601622819900513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685584078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419458895921707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685599922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971851795911789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685615696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217641294002533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685631381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237505078315735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685646883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069425567984581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685661957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09574807435274124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685677126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520587652921677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685692406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442414879798889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685707071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034182608127594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685722876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167093575000763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685737904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257232189178467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685753263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173749178647995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685753264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685753264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685840151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793182730674744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685840152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685840153, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784685855436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109281539916992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685870763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931676834821701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685886259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254061967134476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685901336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025238111615181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685916799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09837029874324799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685931992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166099667549133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685947084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09697774797677994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685962251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957364946603775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685977773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356062650680542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784685993303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904625058174133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686008681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432978719472885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686024106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752524435520172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686039636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176939517259598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686054889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0959811732172966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686054890, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686054890, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686140270, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794694900512695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686140271, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686140271, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784686155823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09394524991512299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686171186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922175854444504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686186574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777156263589859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686201820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010772198438644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686216979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404306650161743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686232850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09538286179304123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686248319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293464362621307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686263960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09848278015851974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686279234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09835398197174072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686294575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439663380384445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686309969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09375689923763275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686325708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10786783695220947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686341442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515213757753372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686357205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028849333524704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686357205, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686357206, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686439664, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7793852090835571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686439665, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686439665, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784686454993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513890534639359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686470453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201021283864975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686485905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052786186337471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686501830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304904729127884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686516854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09672944992780685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686531754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492707788944244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686547074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09564079344272614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686562237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389676481485367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686577298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09790477901697159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686592768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09060725569725037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686608075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038802027702332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686623363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10487190634012222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686638469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388905555009842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686653768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935771465301514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686653768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686653769, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686736299, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799687385559082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686736300, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686736300, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784686751442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1008588895201683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686766272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497195273637772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686781314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09783347696065903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686796399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999680832028389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686811705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109575092792511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686826702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09447021037340164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686841885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434375703334808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686857190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312832146883011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686872196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072551667690277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784686887545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011790484189987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed608892790.log b/recommendation_v4/rcp_logs/gbs_8192/seed608892790.log new file mode 100644 index 000000000..47a0da4ed --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed608892790.log @@ -0,0 +1,1862 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651373285, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651373285, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651394802, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "seed", "value": 608892790, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651394808, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651394818, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651394818, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652000394, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652000395, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652000849, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652111742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14000114798545837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652123962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138068288564682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652136899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868677616119385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652149861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840050995349884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652162999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855326175689697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652176082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138470858335495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652189226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829006254673004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652202465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13928113877773285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652215730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384502798318863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652228833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383925974369049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652242342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861584663391113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652255698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867270946502686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652269292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13832053542137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652282602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13932497799396515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652282603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652282604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652367370, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017393827438354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652367371, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652367371, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652380450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383504718542099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652393624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823029398918152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652407471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383514404296875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652420753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814978301525116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652434319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376705765724182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652447713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379237174987793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652460669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778117299079895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652473929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758443295955658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652488020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785773515701294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652501719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13799405097961426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652515426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13841718435287476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652528950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839800655841827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652542303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13780489563941956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652556085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814930617809296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652556086, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652556087, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652640917, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5019819140434265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652640918, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652640918, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652654821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758760690689087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652668717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381624937057495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652682558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371433436870575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652696250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662269711494446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652709518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778012990951538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652722802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793230056762695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652736094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734310865402222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652750003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674136996269226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652763841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654838502407074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652777488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748766481876373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652791306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370123028755188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652804771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735604286193848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652818734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375727653503418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652832325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752198219299316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652832326, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652832326, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652916810, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5023037791252136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652916812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652916812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652930056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652504980564117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652943605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730089366436005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652957044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365533024072647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652971104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372009664773941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652984814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13725833594799042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652998881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755150139331818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653012809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640475273132324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653026686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373845338821411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653040793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364825814962387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653054860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845239579677582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653068803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13611939549446106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653082623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375799924135208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653096533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493506610393524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653110532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367487758398056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653110533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653110533, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653193705, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5027639865875244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653193706, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653193706, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653207342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603723049163818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653221085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532456755638123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653234543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762368261814117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653248251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431593775749207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653262052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675042986869812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653275737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312450051307678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653289557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546667993068695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653303369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533969223499298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653317259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378367841243744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653330913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351056843996048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653344797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.147623211145401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653358940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549011945724487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653372703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381673395633698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653386310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377456784248352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653386311, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653386311, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653468475, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033531188964844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653468476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653468477, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653482022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352279931306839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653495501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13988736271858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653509147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358812153339386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653522792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314996659755707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653536585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13478609919548035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653550691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391689032316208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653564465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332990974187851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653578254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290834426879883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653592144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13320228457450867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653605797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315799206495285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653619811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119730353355408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653633401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500797748565674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653647201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191528618335724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653661252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882099092006683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653661252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653661253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653742141, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5040301084518433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653742142, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653742143, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653756066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360385596752167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653769985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373993337154388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653783775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583062589168549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653797594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348603069782257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653811557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379128098487854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653825398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12779951095581055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653839059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542883098125458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653853010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294671595096588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653867156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360294699668884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653881491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13247844576835632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653895455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375897705554962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653909396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422346115112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653924002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539814949035645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653938140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327687978744507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653938141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653938141, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654019396, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5049763917922974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654019397, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654019397, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654033692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236111402511597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654047826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13390623033046722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654062142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552922010421753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654076534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543374836444855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654090828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13967913389205933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654104946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307277858257294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654119422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514891266822815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654133527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345253467559814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654147537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328127562999725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654161832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188818097114563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654175746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356024444103241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654190086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364651799201965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654204071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333966702222824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654217921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305338442325592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654217921, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654217922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654300167, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5061982274055481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654300168, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654300168, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654314499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783219456672668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654328659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349833905696869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654343196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353524625301361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654357961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313028186559677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654372228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336992233991623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654386714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340872347354889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654401394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704881072044373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654415912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411536812782288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654429996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331150233745575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654444243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971250712871552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654457939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12786568701267242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654471804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308677077293396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654485799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295679807662964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654499405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248839721083641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654499406, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654499406, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654580447, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074137449264526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654580447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654580448, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654594465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124054670333862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654608181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13605035841464996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654622260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334376037120819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654636518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026803731918335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654650330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282372921705246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654664131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12679721415042877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654678477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12889611721038818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654692430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13239187002182007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654706572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554134964942932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654720761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12507778406143188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654734510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1286332756280899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654748327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13448426127433777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654762316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098640739917755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654776762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920625507831573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654776762, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654776763, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654859244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5089985132217407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654859245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654859245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654873569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031615316867828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654887516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289985179901123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654900881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13307543098926544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654914847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12396544218063354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654929087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12942910194396973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654942639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12600994110107422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654956215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001684844493866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654970021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187302112579346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654983497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13587427139282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654997456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717910647392273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655011195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12370573729276657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655025355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12320242077112198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655039507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12573668360710144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655053574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1288933902978897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655053575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655053575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655138221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5107606649398804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655138222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655138222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655151830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679595291614532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655165917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655762374401093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655179734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357121616601944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655193518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398375928401947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654133677482605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655221514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831061124801636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655235518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12717662751674652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655249477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261729896068573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655263344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14218677580356598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655277663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14094887673854828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655292116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12719126045703888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655306306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12866869568824768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655320486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13252940773963928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655334111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315578818321228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655334111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655334112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655418286, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.512753427028656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655418287, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655418287, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655432870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14116446673870087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655447431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14241275191307068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655461477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542652130126953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655475657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13945145905017853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655490357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14486132562160492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655504519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410072296857834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655519017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12604902684688568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655533257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775141537189484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655547703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175959885120392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655562292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623753190040588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655576748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772843778133392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655590796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13293102383613586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655605266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13426880538463593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655619691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13643088936805725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655619692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655619692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655703501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5148807764053345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655703502, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655703502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655717620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859637081623077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655732006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830192387104034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655746390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13176797330379486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655760682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330135017633438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655774890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372744619846344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655789403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13965904712677002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655803759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306811451911926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655818413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14125308394432068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655832325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1406189501285553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655846715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350088268518448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655860979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12764815986156464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655874919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299593597650528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655888888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341792494058609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655903005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276671439409256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655903005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655903006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655985341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5173585414886475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655985342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655985342, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655999310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13221052289009094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656013280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368718147277832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656027332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336170881986618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656041367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202744722366333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656055578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339954137802124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656069592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330070674419403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656084077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376159965991974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656098323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280895352363586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656112312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13978351652622223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656126789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175027072429657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656141097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880903482437134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656154958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312386840581894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656168819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234595954418182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656182847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359529048204422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656182848, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656182848, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656266187, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5205321907997131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656266188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656266188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656280224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357733190059662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656294119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12297489494085312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656308105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067346811294556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656322239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303381621837616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656336200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522467017173767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656350730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519759476184845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656364938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13552874326705933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656379456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12401732802391052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656393720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104309141635895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656407515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114216923713684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656421759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304718255996704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656435878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337352871894836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656450133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13088461756706238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656464122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13020633161067963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656464123, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656464123, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656548678, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5245648622512817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656548679, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656548680, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656562758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378382295370102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656576954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322588324546814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656591107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343445956707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656605256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304611712694168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656619472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12712818384170532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656633484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212689757347107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656647992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13570064306259155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656662271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297870755195618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656676534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13140693306922913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656691004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13900849223136902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656705342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387465476989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656719054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13732844591140747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656733317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131922647356987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656747334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334579586982727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656747335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656747335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656832452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5298720598220825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656832453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656832453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656846482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122856616973877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656860395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503165543079376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656874506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791635632514954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656888566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332295417785645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656902573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763948738574982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656916615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294354498386383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656930632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12589934468269348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656944499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393073737621307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656958640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13597019016742706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656972890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741081953048706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656987080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345970630645752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657001716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12715543806552887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657015795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064968585968018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657029421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12378666549921036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657029422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657029423, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657115523, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5369929075241089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657115525, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657115525, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657129618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324467808008194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657143820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13262085616588593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657158123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367480039596558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657172348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786937296390533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657186544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363953948020935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657200773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13160909712314606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657214818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723268568515778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657228986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511967658996582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657243014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12476570904254913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657257394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303790807723999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657271629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324956715106964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657286091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342829465866089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657300578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1405745893716812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657314912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361006498336792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657314913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657314913, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657399756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5470276474952698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657399757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657399757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657414730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672739267349243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657429202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315537691116333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657443392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254567980766296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657457673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13917972147464752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657472397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004785776138306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657487279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835090398788452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657501651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515137135982513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657516306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751667737960815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657530810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524393737316132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657545560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13074341416358948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657560049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14077730476856232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657574453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13465183973312378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657588886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291329711675644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657603436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133405864238739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657603437, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657603437, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657688426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5590032339096069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657688427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657688427, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657703242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1422649472951889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657717787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290730118751526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657732967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341543078422546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657747442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308809220790863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657762440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529038429260254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657777117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394442200660706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657791934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417409360408783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657806905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588431477546692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657821581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314810812473297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657836403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13324804604053497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657851221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330728530883789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657866121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731858134269714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657880674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622914254665375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657895329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377558559179306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657895329, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657895330, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657977404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5730664134025574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657977405, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657977405, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657991865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319100558757782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658006644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561823964118958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658021139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13454578816890717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658036140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13233207166194916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658050550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748767971992493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658064930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12982645630836487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658079660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340051144361496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658093794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12454209476709366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658108408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313331127166748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658122845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317623108625412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658137345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169869780540466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658151675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208048045635223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658166076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12947766482830048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658180475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138508796691895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658180476, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658180476, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658261446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5855993628501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658261447, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658261447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658275522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12291616201400757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658290019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12751616537570953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658304369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032039999961853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658318959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338053196668625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658333717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275852799415588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658348344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246491760015488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658363359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315556764602661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658378473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13277390599250793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658393260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769230246543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658407514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12886163592338562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658422246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13417582213878632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658436461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11876536160707474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658450664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12280888110399246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658464944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649144232273102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658464945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658464945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658547256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.598785936832428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658547257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658547257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658561390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226564347743988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658575319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12915216386318207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658589466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297753006219864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658603493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321612775325775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658617658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12301484495401382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658631783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135373055934906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658645822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098646700382233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658659758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312046438455582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658674035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12847277522087097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658688458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1287807822227478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658702591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998475342988968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658717284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325565129518509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658731857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1284189075231552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658746255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12927192449569702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658746255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658746256, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658829799, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6140835881233215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658829800, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658829801, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658843824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13063745200634003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658858447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12066157162189484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658872939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1234724149107933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658887742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1232592985033989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658902306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12481098622083664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658916799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12786990404129028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658931250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11858420073986053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658945898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12400226294994354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658960426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12753300368785858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658974794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261557638645172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658988890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11773308366537094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659003108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12707474827766418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659017218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272622644901276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659031452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1264643669128418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659031453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659031453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659117632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6280779838562012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659117633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659117633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659132383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922008544206619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659147296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1173192486166954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659161910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13155175745487213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659176722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12397246807813644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659191215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12636758387088776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659205545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11910468339920044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659219713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226826161146164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659234154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12379074096679688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659248481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12315414100885391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659263037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11528922617435455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659277294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247343197464943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659291491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12295893579721451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659305833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270066648721695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659320420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11883050948381424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659320420, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659320421, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659404458, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6490457057952881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659404458, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659404459, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659418828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311500310897827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659433096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420612037181854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659447235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316808313131332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659461478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11514115333557129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659475714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160355806350708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659490050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12349960952997208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659504703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11842518299818039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659519588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12179208546876907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659534059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656820237636566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659548927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12026790529489517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659563588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453462392091751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659578250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494127660989761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659592576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213478446006775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659606907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11712580919265747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659606908, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659606908, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659690991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6802730560302734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659690992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659690992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659705328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12144225835800171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659719856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12043830752372742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659734538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12450262904167175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659749073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866102576255798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659763659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520036309957504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659778039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11891253292560577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659792759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995908617973328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659807389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11976273357868195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659821696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997448861598969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659835971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11823613941669464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659850328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11587479710578918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659864458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708100140094757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659878720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141228899359703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659893327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12451628595590591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659893327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659893328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659977550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7147897481918335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659977551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659977552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659992323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12431229650974274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660007356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11736683547496796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660021889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586586385965347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660036974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12172042578458786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660051910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897259414196014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660066767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187243461608887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660080858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392423510551453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660095484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828923434019089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660110190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181662231683731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660125369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627846956253052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660140162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226077452301979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660154858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495795100927353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660169761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12527374923229218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660184684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274682730436325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660184685, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660184685, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660268285, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7546164989471436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660268286, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660268286, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660283176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12470842897891998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660297811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131579652428627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660312508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11499679833650589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660327582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131376251578331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660342247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769627034664154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660357048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11655879020690918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660372204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022048234939575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660387319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966336518526077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660402304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390737444162369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660417654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171571910381317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660432528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423327773809433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660447264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268696933984756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660462019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1232934296131134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660477031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401788145303726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660477032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660477032, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660561328, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7611082196235657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660561329, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660561329, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660576132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11984685808420181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660591011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11766360700130463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660605736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281639337539673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660620684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249017715454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660635329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715711116790771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660650144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134970560669899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660664953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11642353981733322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660679434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10869534313678741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660694154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089162677526474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660708692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076248362660408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660723752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12176716327667236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660738930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492084175348282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660753812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11888004094362259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660769157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11917542666196823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660769158, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660769158, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660851596, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7644199728965759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660851597, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660851598, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660866699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11719263345003128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660881338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11620736867189407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660895906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11513251066207886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660910867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347945034503937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660925709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158698260784149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660940411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100437231361866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660954960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755093932151794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660969400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11475787311792374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660984020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288571566343307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660998540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083982348442078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661012676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530228912830353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661027190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11142877489328384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661041725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880748182535172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661056117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11360067129135132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661056155, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661056156, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661138375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7666370272636414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661138376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661138376, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661152927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410186648368835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661167770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686875134706497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661182716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279323697090149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661197361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895219445228577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661212379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663411021232605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661227509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324037611484528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661242342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127328872680664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661256947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012986302375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661271521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114463210105896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661286033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11047772318124771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661300395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09852208197116852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661314864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128077581524849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661329109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517806559801102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661343352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429532080888748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661343352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661343353, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661426111, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683467268943787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661426112, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661426112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661440494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539735108613968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661454983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198108643293381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661469472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09483291208744049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661483664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039741188287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661498024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181169748306274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661512161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418003052473068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661526555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08836931735277176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661540766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901370644569397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661555098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006865113973618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661569587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779609322547913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661584215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711562633514404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661598564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10031800717115402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661613469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09854386746883392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661628175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671648383140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661628175, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661628176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661710446, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696629762649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661710446, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661710447, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661725045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437379568815231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661739663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137482523918152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661753675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427194088697433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661768386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048821285367012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661782703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0990864709019661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661797178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014898329973221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661811387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633654147386551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661825798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660018771886826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661839931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09713813662528992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661854186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086580827832222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661868512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415586829185486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661882856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629871487617493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661897233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108119435608387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661911657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258253455162048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661911658, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661911658, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661993606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.77080237865448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661993607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661993607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662008062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967239737510681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662022257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09794043749570847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662036786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246726870536804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662051451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451114177703857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662066301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069680750370026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662081016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151731759309769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662095620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104315727949142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662110277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453389585018158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662125178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760864615440369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662139499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483434051275253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662154120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210756748914719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662168100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09393900632858276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662182961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0973895937204361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662197620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728548467159271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662197620, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662197621, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662280816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716514468193054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662280817, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662280817, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662295310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0916917473077774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662310111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426215827465057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662324784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098543033003807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662339671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09707118570804596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662354333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027435213327408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662369040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986372828483582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662383623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970299169421196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662397952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105860024690628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662412498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462680459022522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662427085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035844013094902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662441768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626643896102905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662456066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10474835336208344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662470489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693911671638489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662485152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593495517969131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662485153, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662485153, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662568462, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7722427248954773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662568463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662568463, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662583499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348645597696304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662598485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034751683473587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662612981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499922931194305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662627788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302161425352097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662642541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805673360824585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662657196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960448741912842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662672059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473848879337311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662686332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09613465517759323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662700991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754036903381348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662715758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789915919303894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662730463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11687077581882477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662745419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427178978919983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662760040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648429930210114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488109081983566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662774564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662856513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728497982025146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662856514, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662856514, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662870999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983842819929123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662885536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583968997001648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662899929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650186985731125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662914602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148099064826965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662928960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09164933860301971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662943511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0970887839794159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662957675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351342707872391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662972185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714955627918243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662986375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706751048564911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663000712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037300735712051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663015579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207718402147293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663030380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013123601675034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663045319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946077853441238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663059900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115405336022377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663059901, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663059901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663143372, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732407450675964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663143373, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663143373, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663158360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793280601501465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663173182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192117840051651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663187860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100916713476181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663202747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09164629876613617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663217020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435072869062424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663231292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645205527544022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663246323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997629910707474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663260960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080169677734375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663276031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12168574333190918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663290870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521773993968964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663305763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092955619096756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663320879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12242545187473297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663335733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09913983196020126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663350546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105160042643547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663350547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663350547, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663433388, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773664653301239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663433389, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663433389, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663448201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426076501607895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663463256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11798612028360367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663478149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909633338451385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663493517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283568501472473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663508416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597435593605042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663523511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316286027431488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663538861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618574917316437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663554106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544896870851517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663569131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11450653523206711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663584056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09977230429649353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663599540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12502546608448029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663614983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664557665586472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663630143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139000803232193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663645069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722939670085907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663645070, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663645070, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663730853, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738804221153259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663730854, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663730854, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663745981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365382373332977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663760781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472412198781967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663775618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139731854200363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663790672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11718259751796722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663805607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11385504901409149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663820474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11843079328536987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663835666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266328990459442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663850468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12105007469654083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663865267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600922048091888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663880328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001784354448318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663895201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790004581212997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663910131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10779757052659988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663924986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442648619413376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663940059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1143011599779129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663940060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663940060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664024497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741292119026184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664024498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664024498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664039705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1187518909573555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664054895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107017382979393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664069849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775497555732727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664085257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848681628704071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664100386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019229143857956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664115346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948789864778519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664130326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374341696500778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664145268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842649638652802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664159969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472045838832855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664174439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012504458427429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664189368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360241681337357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664204180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09592613577842712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664218767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351555794477463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664233465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10088913142681122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664233465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664233466, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664316750, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7744770646095276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664316751, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664316751, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664331325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10943292826414108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664345942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006273627281189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664360411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09636594355106354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664374951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834792256355286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664389486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490774899721146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664404016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390347987413406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664418539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356065630912781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664433126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428611189126968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664447775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172651708126068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664462476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12005612254142761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664477235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11677751690149307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664491904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150692611932755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664506947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937343001365662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664521537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12041814625263214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664521538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664521538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664602804, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747105956077576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664602805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664602805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664617994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11072000116109848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664632821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09583042562007904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664647699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413941740989685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664662612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10287206619977951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664677283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616306215524673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664691909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448899865150452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664706803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550379008054733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664721472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119643896818161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664736469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433974117040634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664751407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765153914690018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664766160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533269494771957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664780904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407572031021118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664795305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718140751123428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664809974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201176255941391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664809975, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664809975, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664892244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747551798820496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664892246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664892246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664907019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09901479631662369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664921983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09388746321201324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664936513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961228683590889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664951393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678340703248978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664966103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962446853518486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664981129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09507126361131668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664995902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100123330950737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665010682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09655866771936417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665025758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274805128574371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665040914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768607258796692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665056132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498049855232239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665071059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236633569002151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665085848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141241118311882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665101329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079597473144531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665101331, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665101331, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665184118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749427556991577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665184119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665184119, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665199244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09765296429395676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665214365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675755888223648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665229676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971615463495255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665244403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139184445142746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665259114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935157001018524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665273910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789445042610168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665288606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014095693826675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665303431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587553679943085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665318451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263301432132721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665333070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013115793466568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665347732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09975367039442062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665362304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237877070903778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665376947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848125070333481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665391341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150880977511406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665391342, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665391342, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665474986, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752314209938049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665474987, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665474987, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665489571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920076072216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665504182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685458779335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665518768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09048513323068619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665533374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10106226056814194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665548244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607195645570755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665562888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778345912694931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665577483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974394530057907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665592358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150885581970215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665607368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064822435379028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665622303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10389170050621033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665637372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10229598730802536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665652312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688087344169617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665667512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421838611364365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665682753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881313681602478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665682755, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665682755, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665768363, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752134203910828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665768364, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665768364, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665783457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032046377658844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665798419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025993674993515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665813013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039775162935257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665827438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10233592242002487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665842667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668203979730606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665857258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097745805978775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665872051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09803308546543121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665887126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10227040946483612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665902168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160631477832794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665917040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453545302152634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665931895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040809378027916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665946652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10191112011671066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665961624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299070179462433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665976735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611464083194733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665976735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665976736, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666061552, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755323648452759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666061553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666061553, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666076545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810758173465729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666091791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162180662155151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666107010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363411903381348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666122407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084838420152664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666137916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104619599878788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666152992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244195699691772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666168314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121044516563416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666183577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044707894325256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666199086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296727508306503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666214857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596495121717453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666230319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288488864898682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666245710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581676661968231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666261474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512429475784302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666277063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981675386428833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666277063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666277064, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666360160, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756257057189941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666360161, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666360161, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666375493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104544460773468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666390761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926484525203705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666405895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589398443698883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666420632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034605860710144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666435913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857467353343964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666450624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874970257282257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666465475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728070139884949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666480481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11516472697257996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666495276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383868008852005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666509889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001971885561943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666524429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350283980369568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666539047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208465158939362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666553548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10533906519412994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666567976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652177780866623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666567977, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666567978, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666650229, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758560180664062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666650230, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666650230, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666664887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10170295834541321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666679568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046841949224472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666694561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089654490351677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666709166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421250015497208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666723606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09401123970746994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666738710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119346022605896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666753214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144788026809692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666767664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751379281282425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666782534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026025116443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666797445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334397852420807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666811891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10920944064855576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666826775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428947746753693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666841560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105608731508255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666856313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10261566191911697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666856313, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666856314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666938543, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7757349610328674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666938544, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666938544, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666953591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11287491768598557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666968338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909225791692734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666983060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10969007760286331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666997974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697819292545319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667012436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144148021936417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667026820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10741734504699707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667042340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11800763756036758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667057700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885107517242432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667073038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599753051996231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667088630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1207118108868599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667103757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704424232244492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667118857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098903939127922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667134044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11495348066091537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667149233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974089056253433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667149234, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667149234, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667231636, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758291959762573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667231637, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667231637, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667246893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113309845328331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667262147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10906170308589935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667277378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10998833924531937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667292651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643799602985382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667307771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431558638811111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667323003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307273805141449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667337959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068151518702507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667353100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11466419696807861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667368495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245732009410858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667383979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1198427751660347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667399489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10596095025539398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667414707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11729647219181061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667430070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708973556756973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667445686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828083544969559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667445687, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667445687, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667531513, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759235501289368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667531515, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667531515, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667546950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11929236352443695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667562324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078702062368393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667577594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11929339170455933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667592736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395562440156937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667608033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10735230892896652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667623648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554884910583496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667638898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110508069396019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667654478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633240431547165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667669948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137910932302475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667685132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10583265125751495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667700495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11245761066675186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667715912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652391612529755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667731070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11692974716424942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667746660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468790680170059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667746661, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667746661, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667830805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762800455093384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667830806, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667830806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667846153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949083417654037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667861769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11827143281698227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667877204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038926988840103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667892492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134248748421669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667907651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553449928760529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667922998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10886721312999725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667938502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815735161304474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667954024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931219160556793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667969628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120849996805191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667984983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193620413541794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668000607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979701578617096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668016478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11043700575828552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668032177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163162440061569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668047893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894259810447693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668047893, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668047894, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668129914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766838669776917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668129916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668129916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668145100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11776703596115112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668160348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099121943116188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668175550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062590032815933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668190638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270662605762482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668205837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10569436848163605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668221216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121386289596558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668236134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079118400812149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668250908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687859356403351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668265812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784430265426636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668280639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015927791595459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668295614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549277812242508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668310485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442269593477249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668325517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951066762208939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668340606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544827371835709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668340606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668340607, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668422550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768078446388245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668422551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668422551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668437458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476413160562515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668452644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489574074745178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668467686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061895102262497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668482935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09234614670276642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668498069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408361047506332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668512947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241404175758362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668528068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220953077077866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668543481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11223580688238144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668558632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069001704454422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668573852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031716912984848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668589111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837776213884354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668604565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936125367879868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668620023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725760459899902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668635164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670141875743866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668635165, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668635165, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668717188, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769597768783569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668717188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668717189, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668732170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10882019251585007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668747188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300895571708679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668762024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091187596321106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668776600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09834402054548264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668791754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11541063338518143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668806770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990656167268753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668821650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307599604129791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668836888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841881483793259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668852083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080934926867485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668866907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083945631980896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668881864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097526103258133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668896844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09858580678701401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668911468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678296536207199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668926182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08692122250795364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668926183, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668926183, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669008369, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774516344070435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669008370, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669008370, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669023236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09561176598072052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669038225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137309670448303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669053175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662076622247696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669068057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10113725811243057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669082982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184977948665619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669097953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09055839478969574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669112584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326305776834488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669127366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09682448953390121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669142065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328172147274017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669157135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619624704122543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669172299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996047407388687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669187426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081785187125206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669202434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589540749788284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669217668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030028834939003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669217669, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669217670, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669301680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775658369064331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669301681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669301681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669316763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09576897323131561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669332069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534138977527618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669346912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09471315145492554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669362029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150184482336044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669376963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09733200818300247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669391395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046831518411636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669405928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910652577877045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669420868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055288672447205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669435529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871460288763046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669450281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938157886266708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669465372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613516718149185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669480386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394731163978577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669495212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911142081022263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669510080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812763124704361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669510082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669510082, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669593026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779987454414368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669593027, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669593027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669607626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609585791826248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669622319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448279231786728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669637023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028718426823616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669651697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09925448894500732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669666595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712063312530518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669681562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597675502300262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669697012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432137548923492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669711690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224746912717819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669726496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970684140920639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669741384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098420150578022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669756045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09706452488899231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669770884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168980062007904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669785893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08993085473775864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669800945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631591081619263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669800945, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669800946, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669884488, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783759236335754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669884488, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669884489, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669899230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873980283737183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669914325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1000504344701767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669929343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940285980701447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669944570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851716250181198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669959861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963851422071457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669975132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221179574728012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669990171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011556088924408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670005109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052786260843277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670020404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10097033530473709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670035323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074639856815338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670050536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047364234924316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670065296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745299607515335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670080225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528364777565002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670095416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574135929346085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670095417, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670095417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670180213, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.778720498085022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670180214, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670180214, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670195562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473089665174484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670210630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09772282838821411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670225482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661906003952026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670240707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616790503263474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670255969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197277367115021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670270867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10730931907892227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670285731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097060427069664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670300984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09769213199615479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670316255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500429570674896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670331249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032751053571701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670346462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139202326536179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670361646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935831069946289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670376653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613537579774857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670391538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957922250032425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670391539, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670391539, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670476896, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791900038719177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670476897, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670476897, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670491450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149786621332169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670505918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522475093603134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670520564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09442780911922455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670535588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845425188541412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670550324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016613319516182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670565560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09941837936639786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670580712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706344246864319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670595836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937470942735672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670610977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856827348470688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670626261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873882681131363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670641625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014343872666359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670656397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698838740587234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670671632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160343885421753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670686449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042720079421997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670686450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670686450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670771560, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795014381408691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670771561, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670771561, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670785981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565131902694702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670801318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10450229793787003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670816234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602576285600662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670831521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123049259185791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670846961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671916604042053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670862455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781046003103256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670877528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073506772518158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670892725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717871248722076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670907959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567782819271088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670922980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364636033773422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670938154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807977616786957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670952863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366684943437576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670968156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10559242218732834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670983245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09509117901325226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670983292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670983293, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671067025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7795891165733337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671067026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671067026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671082165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09883357584476471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671097203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436942428350449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671112236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245530307292938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671127439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10463204234838486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671142601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770351439714432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671157628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030281871557236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671172712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778985172510147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671187637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887620806694031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671202712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017929762601852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671217667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007815599441528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671232958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11504516005516052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671248259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977713018655777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671263641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603281110525131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671278807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294102132320404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671278853, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671278854, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671362590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7800170183181763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671362591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671362591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671377910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09589602053165436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671393274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09729047119617462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671408288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212624609470367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671423811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11683442443609238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671438909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253564059734344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671453692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069117486476898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671469064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653717070817947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671484008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1199997290968895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671498930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442742705345154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671514525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863355547189713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671529928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929349809885025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671545100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11009372025728226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671560461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298075318336487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671575805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098053306341171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671575806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671575807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671659163, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7802456021308899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671659164, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671659164, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671674413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755036026239395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671689607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978513211011887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671704948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409157514572144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671720038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980475693941116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671735116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073610857129097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671750397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713132470846176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671765490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171766579151154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671780419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11218655109405518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671795442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11950506269931793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671810507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10845068097114563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671825496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086277961730957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671840707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383087396621704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671856068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202944278717041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671871353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875076055526733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671871353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671871354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671952637, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807974219322205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671952638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671952638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671967662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422436684370041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671982964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11836467683315277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671998231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278647929430008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672013743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291701346635818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672029190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000034213066101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672044299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11898383498191833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672059673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110912024974823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672074436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097634881734848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672089743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004745960235596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672104941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09894969314336777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672120187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001098901033401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672135947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940300673246384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672151484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10342629253864288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672166674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029630899429321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672166675, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672166675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672248097, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780972957611084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672248098, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672248098, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672263224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112589530646801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672278707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12641653418540955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672294185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781095921993256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672309588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117968037724495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672324894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212952435016632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672340478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10873544961214066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11390899866819382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672371756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272364854812622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672387092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660412907600403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672402499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847050696611404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672418107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296059638261795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672433417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249697953462601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672449278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187435686588287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672465139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537877470254898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672465139, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672465140, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672548464, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813300490379333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672548465, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672548465, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672564004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09598948806524277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672579802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11080239713191986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672595607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1213335171341896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672611303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10675707459449768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672627045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079007387161255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672642738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692290961742401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672658219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268346756696701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672673291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306016147136688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672688397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10307762026786804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672703099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775368452072144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672717973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658824443817139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672733068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123857647180557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672748206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09172315895557404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672762856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327904671430588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672762857, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672762858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672844440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810871601104736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672844441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672844441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672859443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979330003261566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672874290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09597344696521759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672888970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291813105344772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672903702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745544731616974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672918312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970255732536316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672933451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513246059417725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672948361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749775916337967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672963321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196999251842499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672978196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993768572807312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672993516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09518342465162277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673008619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09702128171920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673023595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09680649638175964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673038606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198073089122772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673053607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980260580778122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673053608, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673053609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673135605, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7813848853111267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673135607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673135607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673150669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103104367852211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673166050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019618883728981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673181163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316160321235657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673196319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677952319383621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673211604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10141456872224808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673227058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0981704592704773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673242700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799796134233475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673257882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541454702615738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673273087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645154118537903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673288108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579897463321686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673303496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10376352816820145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673318486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604352504014969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673333228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054924800992012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673348237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207927018404007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673348286, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673348287, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673429722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814478874206543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673429723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673429723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673444274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819416493177414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673459076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155417561531067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673474061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602088272571564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673489238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645456612110138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673504021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988641738891602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673518960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10049532353878021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673533818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09614469856023788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673548432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593967139720917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673563032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0984976664185524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673577612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085878521203995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673592565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10113655775785446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673607146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207642614841461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673621864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09871628135442734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673636652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416439920663834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673636695, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673636696, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673718592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7815358638763428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673718593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673718593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673733451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09685229510068893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673748299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081019788980484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673763083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09958788752555847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673777608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192176699638367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673792074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0941922515630722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673806863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678271949291229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673821925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026902124285698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673836738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203984379768372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673851859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037948951125145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673866692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110074073076248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673881906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944395437836647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673897166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014951765537262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673912440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111694946885109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673927313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100254416465759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673927362, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673927363, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674009891, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818500399589539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674009891, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674009892, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674024744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09767629206180573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674039994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726679861545563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674055260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095824301242828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674069838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047060415148735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674084716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10010268539190292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674099520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825184732675552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674114498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10018213093280792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674129743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09755278378725052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674144795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085977852344513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674160060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778588056564331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674175106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09846988320350647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674190213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09325054287910461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674205072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09436933696269989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674219930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546888411045074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674219931, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674219931, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674304376, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7821173667907715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674304377, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674304377, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674319439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244321823120117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674334490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11323723196983337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674349660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012407600879669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674364839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471468418836594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674379840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002057045698166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674394858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021598637104034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674409952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085430458188057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674424871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048776119947433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674439984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581637173891068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674454968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032911598682404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674470093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09693190455436707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674485509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11120373010635376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674500896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185288280248642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674516176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812263190746307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674516177, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674516178, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674599768, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823681831359863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674599769, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674599769, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674615252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11425869911909103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674630250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10147516429424286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674645750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979044646024704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674661010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10073509812355042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674676231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044510006904602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674691587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09864915907382965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674706648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002578049898148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674721991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10581353306770325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674736831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501938313245773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674751704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09308447688817978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674767193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029311865568161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674782317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081934928894043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674797634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09376794844865799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674813070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017812252044678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674813071, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674813072, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674896911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823840379714966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674896912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674896912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674912222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063779890537262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674927432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11392972618341446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674942386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057322680950165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674957599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600695759057999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674972876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018361747264862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674987920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012246385216713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675003007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628045350313187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675017924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813786834478378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675033069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971459150314331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675047952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09626905620098114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675062929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006391793489456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675077751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09687604755163193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675092975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021103709936142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675107900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202256590127945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675107900, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675107901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675192610, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7824775576591492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675192611, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675192611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675207575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181881487369537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675222499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717082768678665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675237331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11194421350955963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675252767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09281563013792038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675268112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169544070959091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675283435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10718944668769836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675298729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11567581444978714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675313947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10073819011449814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675329409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546642541885376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675345024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086815893650055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675360412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09659723192453384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675375463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068684309720993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675390725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983026772737503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675405689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284562408924103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675405690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675405690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675490628, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7827094197273254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675490629, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675490629, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675506018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11007942259311676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675520887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037072166800499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675535569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09212768822908401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675551009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053717344999313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675566162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080736517906189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675581531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107618361711502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675597423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09629315137863159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675613133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10320020467042923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675628647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505464673042297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675644082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788259863853455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675659468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365903377532959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675674936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452673584222794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675690450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11031609028577805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675706069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11329799890518188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675706069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675706069, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675791018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7829299569129944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675791019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675791019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675806201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809475928544998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675821745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502497851848602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675836867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220322012901306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675852328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778848081827164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675867738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138515383005142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675883186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09666728973388672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675898338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10202895104885101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675913365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135919392108917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675928334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109741985797882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675943408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09435955435037613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675958550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103609099984169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675973832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178324371576309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675989191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10122798383235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676004220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11737354844808578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676004221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676004222, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676089245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7830372452735901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676089246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676089246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676104576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978458285331726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676119484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09688183665275574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676134458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616863518953323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676149710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240551620721817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676165048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644410341978073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676180196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09594473242759705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676195493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595989972352982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676210341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070965439081192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676225608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082750022411346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676240787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10434719920158386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676255438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648022592067719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676270967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092873066663742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676286305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09862904995679855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676301834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340783536434174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676301835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676301835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676383933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833932042121887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676383935, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676383935, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676399531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172718554735184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676415116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066383868455887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676430437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563471913337708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676445994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1187838762998581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676461299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163952574133873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676476818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10037223994731903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676492394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699657142162323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676507677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345711350440979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676523118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538486391305923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676538443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678541660308838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676553878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663831233978271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676569391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560205578804016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676584977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677821189165115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544080078601837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676600130, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676600130, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676681671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7835028171539307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676681672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676681672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676697055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147087812423706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676712065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348732769489288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676727408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198625177145004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676742898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11654190719127655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676758563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792148858308792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676774010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476168990135193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676789641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354871302843094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676805095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359565705060959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676820979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11788478493690491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676836527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410354286432266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676851949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610592156648636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676867680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196054518222809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676882860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748454928398132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676898481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578895151615143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676898481, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676898482, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676980118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7838961482048035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676980119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676980119, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676995416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012599766254425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677011081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109076589345932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677026635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11500103771686554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677042195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020353645086288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677058149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857619553804398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677073928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732414573431015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677089505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09853905439376831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677104973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777215659618378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677120628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962862730026245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677136002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545747727155685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677151264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455345898866653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677166811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11049023270606995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677182079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717396229505539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677197906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11226601153612137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677197907, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677197907, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677280283, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845732569694519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677280284, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677280284, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677295729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156948655843735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677311392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557401925325394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677326694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106032058596611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677342100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11539960652589798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677357397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199276894330978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677372889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669138818979263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677388626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09926493465900421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677404016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821489989757538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677419616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725107789039612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677435047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078919842839241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677450647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11726675927639008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677465921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11015457659959793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677481072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251963138580322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677496387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070510745048523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677496388, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677496389, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677578320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837429046630859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677578321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677578321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677593306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547448694705963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677608574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983451455831528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677623704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872343719005585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677639120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679914802312851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677654220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052824854850769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677669375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045001655817032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677685084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622108727693558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677700502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905206203460693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677715716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235379427671432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677730746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09367325901985168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677745979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626666784286499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677761102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954246878623962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677775958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889503568410873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677790913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723663121461868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677790913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677790914, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677872479, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7839300036430359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677872480, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677872480, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677887549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090027749538422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677902893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09317588061094284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677918606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691544622182846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677934052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084933653473854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677949339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10771240293979645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677964713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225724965333939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677979871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09513312578201294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677995301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600251704454422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678010720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051260381937027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678025966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997620224952698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678041233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09760033339262009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678056956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09860437363386154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678072347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110815778374672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678087574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09872838854789734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678087574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678087575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678168408, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7844783663749695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678168409, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678168409, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678184083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571573674678802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678199507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09555906057357788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678215057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292669385671616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678230830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554927051067352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678246515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10231658071279526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678262142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102536991238594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678277713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10654932260513306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678293166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09491526335477829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678308748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352812498807907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678324240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368206351995468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678339218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027660220861435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678354695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177940130233765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678369474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282102227210999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678384302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011575385928154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678384303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678384303, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678466820, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7842319011688232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678466821, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678466821, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678482079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002863273024559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678497271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907042980194092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678512422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10127788782119751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678527154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137521475553513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678542268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873256832361221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678557192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001386046409607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678572028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09680439531803131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678586714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011087357997894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678601701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1029568687081337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678616799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758139938116074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678631653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036047413945198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678646417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739327222108841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678661624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137569904327393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678676332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09506594389677048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678676335, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678676335, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678759992, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7845577597618103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678759993, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678759993, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678775137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09424804151058197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678789654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879379719495773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678804426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09608615934848785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678819346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09957832843065262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678834077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037931814789772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678849423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09490721672773361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678864218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10168324410915375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678879368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09768480807542801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678894359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09778311103582382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678909260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388895124197006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678924416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09305758029222488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678939875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684406757354736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678955324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458049923181534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678970746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725986212491989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678970747, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678970747, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679053349, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.784905731678009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679053349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679053350, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679068603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490144789218903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679083701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10098166763782501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679098640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476893931627274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679114057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10256075114011765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679128693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09610196948051453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679143411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511057823896408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679158713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09461726993322372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679173813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355739831924438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679189048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09801045805215836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679204887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08961890637874603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679220553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048855096101761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679235906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453606396913528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679251152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418515652418137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679266266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087312400341034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679266266, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679266267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679346989, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7849149107933044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679346989, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679346990, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679362208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955613315105438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679376968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518582165241241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679392083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09749330580234528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679407362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019289702177048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679422817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017166286706924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679437998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0943777933716774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679453385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435215383768082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679468934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024128794670105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679483941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995562583208084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679499368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946498811244965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679514260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09736167639493942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679529676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09951602667570114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679544875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09934127330780029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679559962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532228648662567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679559963, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679559963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679642452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7846201658248901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679642453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679642453, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679657784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1009301245212555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679673074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09493880718946457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679688102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09804070740938187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679702827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825282335281372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679718112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028633788228035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679733239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08705287426710129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679748475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605185478925705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679763749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09289876371622086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679778998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036633774638176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679794224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09509050101041794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679809177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09429726004600525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679823881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288754105567932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679838945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11177383363246918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679853849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09754238277673721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679853850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679853850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679938056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7854417562484741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679938057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679938057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679952742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0951664075255394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679967910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09560933709144592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679982704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040736198425293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679997425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169914364814758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680012477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303870588541031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680027662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051115021109581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680042811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081317663192749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680057570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155603677034378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680072373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10784298926591873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680087352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09909893572330475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680102298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042483896017075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680117197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09602565318346024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680132160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09412126988172531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680147057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758287459611893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680147057, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680147058, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680230745, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7854156494140625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680230746, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784680230746, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed666410936.log b/recommendation_v4/rcp_logs/gbs_8192/seed666410936.log new file mode 100644 index 000000000..8719616f2 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed666410936.log @@ -0,0 +1,1818 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651470907, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651470908, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651492049, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "seed", "value": 666410936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651492052, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651492053, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651492053, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651492053, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651492054, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652395120, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652395121, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652395456, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652571815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779228925704956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652584317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13948743045330048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652597193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861797749996185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652610158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13914042711257935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652623332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13889621198177338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652636498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13873125612735748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652649661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13904644548892975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652662809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885930180549622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652676207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890239596366882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652689409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13903111219406128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652703014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13884574174880981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652716429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387496292591095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652730201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13901670277118683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652743562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13824324309825897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652743563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652743563, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652928256, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49880778789520264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652928257, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652928257, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652941120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388450562953949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652954241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882307708263397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652967663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386668086051941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652981058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389031559228897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652994642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389792114496231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653008478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387631893157959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653021582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387048214673996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653034945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387728452682495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653048944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863505423069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653062527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386292725801468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653076397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875772058963776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653089857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861820101737976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653103364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863421976566315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653117252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13868071138858795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653117252, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653117253, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653202264, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49902045726776123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653202265, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653202266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653215990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838881254196167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653229898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138384610414505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653243818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822486996650696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653257724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787609338760376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653271072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13839542865753174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653284452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854840397834778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653297947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788393139839172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653312289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787472248077393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653326278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13758111000061035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653340014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138236403465271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653353979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13785718381404877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653367661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381261646747589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653381740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380717158317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653395242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13806062936782837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653395242, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653395243, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653479492, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4992886781692505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653479492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653479493, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653492790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739073276519775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653506413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777832686901093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653519990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728058338165283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653534367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757362961769104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653548334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378859281539917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653562370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13791893422603607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653576271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372217833995819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653590060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759775459766388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653604251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370912790298462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653618245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867442309856415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653632018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682374358177185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653645722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803166151046753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653659634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359868198633194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653673876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735145330429077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653673876, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653673877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653757984, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49963000416755676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653757985, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653757985, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653771630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366024613380432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653785547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615527749061584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653799250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13781486451625824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653813393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519734144210815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653827712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368628293275833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653841618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420552015304565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653855637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592469692230225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653869363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358049362897873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653883276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13790567219257355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653897067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13553261756896973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653910790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1457153558731079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653924902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357528120279312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653938644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344693899154663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653952575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795089721679688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653952575, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653952576, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784654036000, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5000606179237366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784654036001, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784654036002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784654049468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565371930599213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654062989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13949376344680786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654076595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13622674345970154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654090092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382376730442047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654103699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532288372516632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654117411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895002007484436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654131054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370881974697113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654144818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342763483524323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654158419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365617394447327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654171832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13208380341529846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654185493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173159956932068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654198870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350904256105423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654212514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13260327279567719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654226537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387495994567871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654226538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654226538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654307975, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006159543991089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654307976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654307977, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654321829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397163152694702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654335558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369858264923096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654349483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594010472297668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654363176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356057971715927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654376805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735611736774445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654390498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12852488458156586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654403887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13546152412891388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654417513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297309249639511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654431419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366883993148804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654445462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328895092010498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654459267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370639085769653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654473363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471965491771698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654487863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353222131729126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654501873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338682621717453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654501874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654501874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654584974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014109015464783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654584976, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654584976, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654599165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327541172504425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654613232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395115733146667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654627478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532766699790955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654641939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541509211063385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654656229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13939908146858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654670345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334255337715149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654684950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532161712646484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654698771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335832178592682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654712653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312476873397827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654726781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13199077546596527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654740597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588209450244904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654754864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362818956375122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654768740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334420144557953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654782497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13072247803211212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654782498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654782498, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654867491, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025701522827148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654867492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654867492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654882370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377616673707962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654896666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13482236862182617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654911120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540126383304596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654925843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314469277858734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654940241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13353198766708374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654954856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431328535079956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654969596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686446845531464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654984343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396470248699188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654998973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306252658367157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655012978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13007725775241852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655026484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765121459960938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655040318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13298562169075012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655054350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12970805168151855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655068010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12482650578022003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655068010, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655068011, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655151843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5037841200828552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655151844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655151845, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655165795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116581737995148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655179652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599827885627747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655193741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310830295085907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052485883235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655221706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283496618270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655235846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268254965543747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655250101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902475893497467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655263823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324000060558319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655277782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13560549914836884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655291871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12516631186008453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655305680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12871238589286804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655319607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457223773002625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655333832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104213774204254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655348370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935864925384521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655348371, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655348371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655430030, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053304433822632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655430031, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655430031, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655444602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067108392715454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655458656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12917710840702057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655472218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329631358385086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655486459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12404709309339523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655500739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12960034608840942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655514485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.125852569937706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655528285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004086911678314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655542360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12191492319107056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655556018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13577960431575775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655570198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711890578269958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655584186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12378627806901932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655598575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12339725345373154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655612804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12577466666698456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655627003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12898333370685577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655627003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655627003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655709433, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5071465373039246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655709434, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655709434, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655723179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654941320419312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655737926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655363023281097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655751940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13572977483272552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655765917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399485170841217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655780026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675682246685028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655794087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833382725715637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655808088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12742003798484802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655822159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12632563710212708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655836345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223116636276245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655850870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14084377884864807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655865497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12718668580055237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655880008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829449772834778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655894416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325046420097351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655908190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167470693588257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655908190, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655908191, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655991644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090919733047485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655991645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655991645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784656006005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1413619965314865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656020514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14226390421390533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656034550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540548086166382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656048788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13947172462940216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656063294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14475145936012268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656077330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14104220271110535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656091684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583285570144653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656105953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760921359062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656120399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13172399997711182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656134885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13613325357437134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656149317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775093853473663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656163594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310843706130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656178794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457806408405304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656193753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654445111751556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656193753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656193754, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656276563, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5111192464828491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656276565, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656276565, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656290989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857316970825195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656305862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823062181472778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656320850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13173167407512665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656335798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287745416164398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656350744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383270800113678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656365887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13987033069133759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656380872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326596587896347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656395904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1412842720746994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656410163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410370171070099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656424953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13499243557453156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656439468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12782010436058044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656453567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301051527261734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656467717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13420256972312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656481956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12763676047325134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656481956, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656481957, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656565023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5134171843528748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656565025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656565025, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656579227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323276311159134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656593285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368236392736435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656607364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354766368865967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320073902606964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656635697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341448575258255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656649858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341209292411804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656664368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393601775169373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656678814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289609551429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656692970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13978737592697144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656707955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195909559726715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656722477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12879972159862518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656736722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121351599693298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656751101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323058307170868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656765513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13584312796592712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656765513, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656765514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656849255, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5164732933044434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656849256, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656849256, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656863782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348668813705444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656878133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12322060018777847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656892435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13071495294570923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656906949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13047625124454498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656921250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525867462158203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656936177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135297492146492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656950637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550104200839996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656965452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12408757209777832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656979861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097259402275085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656993887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118579983711243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657008618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304195374250412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657022797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133245050907135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657037103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13086554408073425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657051095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035528361797333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657051096, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657051097, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657136617, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5203675627708435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657136618, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657136618, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784657150947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779966533184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657165218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242554664611816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657179419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365121185779572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657193669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043734431266785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657207908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271987110376358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657222044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232997059822083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657236733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355055570602417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657251109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328631043434143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657265527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314629763364792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657280147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906662166118622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657294628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394209742546082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657308597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751757144927979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657323480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13188093900680542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657337997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333867460489273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657337997, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657337998, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657422658, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5254380702972412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657422659, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657422659, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657437080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13143497705459595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657451476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510435819625854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657465921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377696692943573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657480494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341769576072693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657494784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12741996347904205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657509219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941165268421173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657523626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12609319388866425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657537892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1242847815155983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657552582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641062378883362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657567298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373344510793686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657581979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13467484712600708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657597184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270236372947693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657611938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096272945404053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657626197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12364225089550018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657626198, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657626198, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657713072, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5326540470123291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657713072, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657713073, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657727795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270393013954163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657742202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13289867341518402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657756337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338193714618683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657770471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797368109226227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657784618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13640348613262177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657798750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149426877498627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657812740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723895907402039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657826788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352739930152893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657840709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12471026182174683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657854882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13052181899547577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657868939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324790120124817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657883335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435068726539612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657897784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1405450701713562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657912082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362663358449936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657912082, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657912083, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657996358, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.543735682964325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657996359, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657996359, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784658011225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665813207626343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658025650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13180691003799438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658039787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13278540968894958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658054352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909024000167847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658069297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301417052745819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658084523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859005272388458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658099072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513141870498657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658113903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137381911277771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658128465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13507120311260223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658143429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13059453666210175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658158237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.140670046210289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658173046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475002348423004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658187907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292363554239273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658202849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337688148021698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658202850, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658202850, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658287306, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.557884693145752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658287307, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658287308, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658302481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14210940897464752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658317248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1328498274087906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658332734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334797739982605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658347661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100557029247284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658362942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539913296699524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658377956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382643461227417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658392592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13422055542469025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658407252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569436967372894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658421615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13138045370578766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658436207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303789496421814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658450742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13302738964557648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658465359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371019333600998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658479712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360195428133011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658494457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756565749645233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658494458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658494459, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658578091, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5751428604125977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658578092, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658578092, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658592467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181172311306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658607028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561171293258667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658621303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13432259857654572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658636094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322411745786667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658650360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374034434556961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658664614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1297641545534134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658679849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395215570926666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658694324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12427052110433578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658708960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112297654151917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658723550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1316249519586563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658738149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124796748161316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658752625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318431943655014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658767223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922298908233643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658781658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129569590091705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658781659, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658781659, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658864930, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5915071964263916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658864931, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658864931, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658879113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12284853309392929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658893568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273595541715622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658907988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12971799075603485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658922571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397392630577087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658937405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13171827793121338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658952026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224258691072464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658967019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308584213256836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658981746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322462260723114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658996343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686257600784302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659010577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12850356101989746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659025531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332244724035263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659039873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11819128692150116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659054234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12186399102210999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659068595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12564636766910553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659068596, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659068596, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659152534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6068456768989563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659152535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659152535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784659166641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12183574587106705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659180798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275227814912796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659194950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12874436378479004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659208978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13201232254505157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659223253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169621884822845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659237527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494938611984253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659251631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294204145669937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659265626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12129330635070801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659279981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692409753799438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659294514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12725569307804108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659308826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11829659342765808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659323540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121651113033295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659337955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12661416828632355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659352498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1283707171678543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659352498, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659352499, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659437443, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6218957901000977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659437444, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659437444, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659451325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12806232273578644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659466135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11850620061159134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659480774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12207120656967163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659495681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12105769664049149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659510225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12296497076749802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659524587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12535502016544342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659538828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1158817857503891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659553137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12181994318962097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659567419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12445539981126785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659581777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1226082295179367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659596052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479179561138153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659610457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12311818450689316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659624794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093001589179039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659639431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12340936809778214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659639432, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659639432, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659721823, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6370042562484741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659721825, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659721825, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659736663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11570336669683456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659751444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316894739866257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659766001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12788832187652588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659780922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12060708552598953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659795430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1223517656326294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659809832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11443130671977997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659823827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812615394592285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659838898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12098999321460724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659853322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11907535791397095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659867886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119735836982727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659882252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11921177804470062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659896630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1205078512430191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659910956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224905475974083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659925581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622431129217148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659925582, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659925582, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784660007440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6873796582221985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784660007441, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784660007441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784660021885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11849922686815262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660036337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096729263663292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660050919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837104171514511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660065283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11301089078187943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660079704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10746709257364273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660094257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12077423930168152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660109048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407039314508438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660124140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11886944621801376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660138772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10155009478330612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660153714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192193552851677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660168490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235448718070984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660183141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139369755983353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660197678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757995396852493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660212457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607929319143295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660212458, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660212458, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660294724, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7329897880554199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660294725, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660294725, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660309076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784987896680832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660324048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11839611083269119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660338948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12128496915102005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660353704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349538177251816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660368687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206412315368652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660383221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592396348714828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660398096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11814191937446594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660412868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711037158966064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660427630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070265993475914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660442276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1164742261171341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660456874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131451204419136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660471416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489156633615494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660485742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111692562699318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660500566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12406039237976074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660500567, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660500567, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660582195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7459519505500793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660582196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660582196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660597082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12257744371891022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660611896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11508463323116302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660626484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597990244626999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660641457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11944083869457245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660656178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10660254955291748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660670947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021783947944641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660685193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11121246963739395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660700183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762687027454376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660714887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963212698698044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660730145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526717990636826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660745047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12189017981290817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660759898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11434542387723923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660774993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1260741949081421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660789931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102049052715302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660789932, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660789933, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660871767, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7539815306663513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660871768, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660871768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660886781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1249026358127594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660901492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215712130069733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660916231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536817252635956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660931266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179704964160919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660945990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11770318448543549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660960909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681848764419556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660976129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12079127132892609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660991308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11989577114582062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661006362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472642421722412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661021851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121939942240715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661036864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11582055687904358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661051634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361641436815262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661066497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12377186864614487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661082182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11350269615650177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661082182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661082183, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661163955, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.756658136844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661163956, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661163956, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784661178967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194806769490242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661194094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828625202178955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661209046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133471876382828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661224115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339270323514938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661238846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704877227544785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661253686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11307384073734283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661268585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1165546104311943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661291934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11015934497117996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661306634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926542431116104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661321113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690488666296005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661335984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12241749465465546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661351005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509604960680008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661365847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12082305550575256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661381053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1197979599237442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661381053, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661381054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661465214, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7581610083580017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661465215, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661465215, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661480307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11795932054519653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661495088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11825006455183029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661509925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11518354713916779, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661525432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11489858478307724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661540273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279609262943268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661555171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316548496484756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661569926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11815165728330612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661584647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536318063735962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661599473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273793339729309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661614346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218784958124161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661628920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665574669837952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661643892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298874020576477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661658771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098573356866837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661673632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11400215327739716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661673679, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661673679, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661758175, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.758251428604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661758176, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661758176, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661773090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10458837449550629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661788291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106508769094944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661803530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11462503671646118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661818281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983408987522125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661833487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816118866205215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661848760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433222889900208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661863681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284468322992325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661878388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281328648328781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661893259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639093607664108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661907694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110984668135643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661922115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005333885550499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661936661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345105618238449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661950913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733485221862793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661965319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565314441919327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661965320, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661965320, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784662052395, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592753171920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784662052396, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784662052396, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784662066686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612494498491287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662081191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296772956848145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662095844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09609023481607437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662110275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414819419384003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662124696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11256776005029678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662139041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465966165065765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662153549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0893137976527214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662167891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903589427471161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662182343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200699418783188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662196906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659587174654007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662211523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074209213256836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662225897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007145568728447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662240830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036127269268036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662255617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783921182155609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662255618, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662255618, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662342076, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7607845067977905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662342077, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662342077, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662356776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11738960444927216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662371549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117834746837616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662385649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10455185174942017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662400916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725414752960205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662415475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09990319609642029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662430373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027659997344017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662444866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662689059972763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662459611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793192684650421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662474226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811818599700928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662489117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108811154961586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662503731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059032529592514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662518249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704021900892258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662532753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859429836273193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662547452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311278492212296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662547453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662547453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662632554, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7640342116355896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662632555, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662632555, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662647193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11090485751628876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662661574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10008399188518524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662676408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248935967683792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662691503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679817944765091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662706546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145040392875671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662721609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408505588769913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662736460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111231356859207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662751380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509157925844193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662766419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752663761377335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662781199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605624318122864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662796071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259784549474716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662810225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.095668725669384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662825285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09775999933481216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662839962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704775899648666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662839962, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662839963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662923798, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651156187057495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662923799, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662923799, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662938105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09220807254314423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662952609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501605272293091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662966972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103169322013855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662981592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09801575541496277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662995938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11063049733638763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663010380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110953241586685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663024691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09799806028604507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663038790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105217039585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663053175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610555857419968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663067396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494839400053024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663081834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853607952594757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663095896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055726557970047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663110159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982198715209961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663124690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674373805522919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663124690, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663124691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663207440, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7653131484985352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663207440, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663207441, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784663222290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10523755848407745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663237224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039656326174736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663251767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10554816573858261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663266489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10416459292173386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663281180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827622026205063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663295731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10086065530776978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663310598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613524168729782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663324702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606652706861496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663339740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863800346851349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663354978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940418392419815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663369737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168668195605278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663384627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454236507415771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663399290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608192324638367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663414038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11496567726135254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663414039, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663414039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663496813, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7665496468544006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663496814, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663496814, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663511351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126189142465591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663526218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649172753095627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663540727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841399431228638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663555594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207924246788025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663570198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09170478582382202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663584820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889429807662964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663599172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425464063882828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663613954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770843923091888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663628408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085488349199295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663642897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132057964801788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663658042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388225108385086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663673167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066469758749008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663688467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066943407058716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663703355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111733578145504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663703356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663703356, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663785928, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7667439579963684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663785929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663785929, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663800944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858535021543503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663815946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190250724554062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663830722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123606562614441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663845600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0921362116932869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663860071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481524258852005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663874533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075633093714714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663889960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115265041589737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663904889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190645396709442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663920327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12218473851680756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663935480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10525549948215485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663950509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219148337841034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663965702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12333056330680847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663980857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961844235658646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663996035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099602282047272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663996036, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663996036, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784664078284, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.766491174697876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784664078285, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784664078285, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784664093392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417351871728897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664108580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11911449581384659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664123742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10952464491128922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664139218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10659032315015793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664154206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09748710691928864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664169499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294085532426834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664184914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665574669837952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664200218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766849666833878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664215344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682768911123276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664230236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973832219839096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664245538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247454434633255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664260727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898815095424652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664275671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11765473335981369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664290563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810019820928574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664290563, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664290564, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664373764, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7671238780021667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664373765, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664373765, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664388935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142413467168808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664403846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153440997004509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664418714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091616004705429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664433754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11868502199649811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664448439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474284529685974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664463106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11948350071907043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664477971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273801028728485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664492299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246906757354736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664506821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11650751531124115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664521756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088525503873825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664536242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897333920001984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664551026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907216370105743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664565765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605720967054367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664580571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11506200581789017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664580572, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664580572, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664662748, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682990431785583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664662749, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664662749, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664677918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200690045952797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664692940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792220383882523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664707612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087208241224289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664722952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802231729030609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664737923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11188586056232452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664752908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106669157743454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664767814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463767290115356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664782771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10850857198238373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664797846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708138346672058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664812371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107002407312393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664827545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10524415224790573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664842384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09747777134180069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664857133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489047318696976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664871820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152829438447952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664871821, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664871822, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664955114, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7685767412185669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664955115, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664955115, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664969785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10993931442499161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664984570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232682526111603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664999090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09663359820842743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665013575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898790508508682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665028206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11565335839986801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665042826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506856441497803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665057383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597730427980423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665072010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483748465776443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665086526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244082659482956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665101201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1206861361861229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665115942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750295758247375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665130630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251644253730774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665145546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096515715122223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665159964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106253206729889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665159964, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665159965, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665244310, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7692006826400757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665244311, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665244311, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665259321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189648509025574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665274131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09741243720054626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665288820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10444527119398117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665303697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449115186929703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665318335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985074937343597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665333409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631248354911804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665348017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071184054017067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665362729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11280123144388199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665377853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528392344713211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665392839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084434762597084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665407638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562460869550705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665422481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581757664680481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665437131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706445574760437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665451912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283046215772629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665451913, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665451913, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665535011, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696719765663147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665535013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665535013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665549736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1004445031285286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665564735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09530457854270935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665579443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09644866734743118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665594376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09748273342847824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665608952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09694258123636246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665623732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09542427211999893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665638376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048892349004745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665653189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717489033937454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665668235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324341058731079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665683053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954219102859497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665698152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517145693302155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665712897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043163388967514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665727763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453254520893097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665743236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11198519915342331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665743237, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665743237, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665825621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704042792320251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665825621, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665825622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665840636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09912067651748657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665855599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789385437965393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665870773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102660521864891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665885496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171958059072495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665900353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10143306106328964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665915234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818132013082504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665930015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146466434001923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665944840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685279965400696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665960064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10302173346281052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665974947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016290932893753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665989814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003422737121582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666004574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306209325790405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666019325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039214581251144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666033989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488932371139526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666033990, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666033990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666117318, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770160436630249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666117319, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666117320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784666132304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123161762952805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666146855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750316083431244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666161730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09172645211219788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666176298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357411950826645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666191097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705558955669403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666205652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848674923181534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666220515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11107546836137772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666235542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421902686357498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666250459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10021824389696121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666265213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492802411317825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666280290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10381156206130981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666294980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10919744521379471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666310018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643143206834793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666325187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094140037894249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666325187, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666325188, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666410700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7711884379386902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666410701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666410701, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666425716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323208570480347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666440774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047891303896904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666455594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10452781617641449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666470148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265380889177322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666485517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835323482751846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666500263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10144633054733276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666515174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09906265884637833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666530444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247892886400223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666545551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229700595140457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666560652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515496879816055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666575733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10410546511411667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666590589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322032868862152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666605708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051945835351944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666620638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107624851167202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666620639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666620640, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666703704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712177634239197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666703705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666703705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666718702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089906394481659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666733821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219824314117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666748499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10451081395149231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666763199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10140541940927505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666778115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544466972351074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666792634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265996843576431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666807437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172293335199356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666821878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035771667957306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666836549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346019059419632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666851600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697108507156372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666866420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340877622365952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666881139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10492032021284103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666896012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589563846588135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666910938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935091435909271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666910938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666910939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666995481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7715105414390564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666995481, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666995482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784667010125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10263223946094513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667024829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992978721857071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667039565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712939500808716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667053837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117913037538528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667069372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899879038333893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667084248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1102331131696701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667099437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076979786157608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667114807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11592614650726318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667129802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147426888346672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667144607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102909058332443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667159510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439550876617432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667174344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035861000418663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667189014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574014484882355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667203749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740181058645248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667203750, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667203750, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667288200, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.771694004535675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667288201, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667288201, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667303054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023353636264801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667317820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138408839702606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667333056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956613719463348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667348038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10546864569187164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667362778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09475858509540558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667378049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135205626487732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667392594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1020771935582161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667407370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827904939651489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667422375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10343454778194427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667437486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446037352085114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667452283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11026474088430405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667467336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467533558607101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667482153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10686740279197693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667497326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10335944592952728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667497327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667497327, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667580245, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719758749008179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667580246, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667580246, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667595373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11351846903562546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667610388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971075296401978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667625584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004337668418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667641095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785451531410217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667655750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10185644030570984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667670321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828061401844025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667686136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11805924773216248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667701281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935915261507034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667716314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662132292985916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667731621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12275488674640656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667746548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792817920446396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667761453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127591133117676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667776555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624860018491745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667791460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104358658194542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667791460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667791461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667874026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7717100977897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667874026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667874026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667889084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402764171361923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667904190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983198881149292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667919085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203627288341522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667934001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676795244216919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667948736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446267575025558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667963750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453354358673096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667978573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743757337331772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667993406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545392870903015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668008608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11222202330827713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668023815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12077335268259048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668039309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585561394691467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668054449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11792152374982834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668069477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870641469955444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668084723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1196366474032402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668084724, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668084724, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668167405, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723491191864014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668167406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668167406, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784668182471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032903730869293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668197530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848748683929443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668212339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11990504711866379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668226975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140122041106224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668241797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829805582761765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668257267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564009636640549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668272122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11308622360229492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668287532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1176062822341919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668303087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374222487211227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668318306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619503259658813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668333589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11432678252458572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668348918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629469156265259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668364159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714635789394379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668379747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10595451295375824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668379748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668379748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668463861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725483775138855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668463861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668463862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668479274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029566079378128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668494947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12126728147268295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668510423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11037356406450272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668525774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449411511421204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668540902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11647981405258179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668556371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000800132751465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668572072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085960790514946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668587510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11040613800287247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668603236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128082349896431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668618565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132301390171051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668634041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050301045179367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668649751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148319393396378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668665147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309029161930084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668680346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938755422830582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668680347, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668680347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668763777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7727013230323792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668763778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668763778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668778895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812707781791687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668794275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081554740667343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668809004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094754189252853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668823752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323517769575119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668838767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600966215133667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668853912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199293285608292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668868675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895495116710663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668883486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901045054197311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668898340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989181324839592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668912912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228480398654938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668927616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575974732637405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668942163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509226471185684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668957121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002559959888458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668971859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168760359287262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668971860, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668971860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784669055609, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731391191482544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784669055610, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784669055611, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784669070455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11523551493883133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669085334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063557043671608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669100022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10146715492010117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669114919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09283459931612015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669129612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1135353296995163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669144233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10571283102035522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669159139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303918272256851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669174304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288771778345108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669189247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725066065788269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669204253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436973720788956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669219292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946916788816452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669234525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311660706996918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669249793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849756002426147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669264846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764004290103912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669264847, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669264847, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669348594, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731704115867615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669348595, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669348595, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669363907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053189635276794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669378980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043868437409401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669393626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109109967947006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669408214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964701533317566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669423471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607228219509125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669438360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11070858687162399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669453150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104305699467659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669468333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817565023899078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669483261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858042538166046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669497873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912565886974335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669512738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099584400653839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669527403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994933694601059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669541833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824538767337799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669556378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08761264383792877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669556378, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669556379, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669638197, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733290791511536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669638198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669638198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669652983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09594782441854477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669667740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110028326511383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669682484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935529321432114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669697256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209225118160248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669712138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247208923101425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669727122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09080001711845398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669742008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10369621217250824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669756693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09844737499952316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669771167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395567119121552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669786220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727105289697647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669801365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118700355291367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669816400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895995795726776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669831317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647457093000412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669846366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429994761943817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669846367, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669846368, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669929637, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7735073566436768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669929638, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669929638, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669944687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09705324470996857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669959984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535910725593567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669975042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09609873592853546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669990192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340158641338348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670005386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0988369807600975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670020171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156817734241486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670034868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953794419765472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670050068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10200905054807663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670064787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10925847291946411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670079716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11028218269348145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670094913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697904974222183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670110016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529898852109909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670124927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074420064687729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670139763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896743088960648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670139764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670139764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670225760, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736728191375732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670225761, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670225761, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784670240541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763110220432281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670255236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591668635606766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670270042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413218289613724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670284989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061998665332794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670300129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723122954368591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670315331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678810834884644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670330998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558079183101654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670346014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433819890022278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670360988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044807732105255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670376288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0994785726070404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670391057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0980367586016655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670406046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306376963853836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670421210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09150764346122742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670436525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811539739370346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670436526, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670436526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670521828, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738646864891052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670521829, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670521829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670537001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10035301744937897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670552364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10173335671424866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670567400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920316189527512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670582769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027022451162338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670598101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10178069770336151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670613413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309427231550217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670628554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10310270637273788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670643596, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610523819923401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670658722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174015909433365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670673404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457791924476624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670688725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10157460719347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670704185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866563767194748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670719341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608558356761932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670734572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739417374134064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670734573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670734573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670820010, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773925244808197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670820011, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670820012, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670835272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739009827375412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670850478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09876921027898788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670865414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708044469356537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670880705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743719339370728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670895983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348626226186752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670910625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743443667888641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670925425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0975453108549118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670940458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827112406492233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670955277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057027131319046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670970065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539416968822479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670985032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318400710821152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670999815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071106791496277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671014707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725880414247513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671029704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10074993222951889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671029705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671029706, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671114776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7741949558258057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671114777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671114778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784671129601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440068691968918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671144415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580072551965714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671159389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09506867825984955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671174661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935220330953598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671189491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336247086524963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671204866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042334347963333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671219853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714592784643173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671234914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004868149757385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671249917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821831226348877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671264812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085000842809677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671279998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349234193563461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671294740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09942512959241867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671309902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228521913290024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671324673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219429969787598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671324674, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671324675, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671411511, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742371559143066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671411512, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671411513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671426034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622715204954147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671441622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516694188117981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671456651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634972900152206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671471910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099711805582047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671487322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687929391860962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671502676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892698913812637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671517687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868561267852783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671532877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819488972425461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671548051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761020332574844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671562983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035892516374588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671578116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896500945091248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671592958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350484400987625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671608220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674640536308289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689601510763168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623297, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671623298, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671709221, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774358868598938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671709222, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671709222, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671724310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966647624969482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671739460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561797767877579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671754635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023624837398529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671769901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10472842305898666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671785117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10856954753398895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671800185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12469470500946045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671815374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963567346334457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671830526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943906217813492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671845859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330919712781906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671861073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253644526004791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671876404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509773880243301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671891696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09941034764051437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671907204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726025700569153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671922539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501381754875183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671922584, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671922585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784672008766, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774552583694458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784672008767, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784672008768, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784672024291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907767921686172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672039608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857383370399475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672054826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298898607492447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672070295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11877603083848953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672085440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427938938140869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672100246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11146964132785797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672115926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758944600820541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672131314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12016630172729492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672146563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10598354786634445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672162139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881493985652924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672177607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234850436449051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672192863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088250577449799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672208232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11366929858922958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672223534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11203683912754059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672223535, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672223535, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672307234, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746077179908752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672307235, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672307235, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672322611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074453815817833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672337984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079395562410355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672353484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148681640625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672368793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083238571882248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672384225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898153483867645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672399741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844742506742477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672415169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172275453805923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672430410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11249790340662003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672445810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12111125141382217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672461282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046586185693741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672476568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10888776928186417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672492149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503049194812775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672507624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442622750997543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672523123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10936161875724792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672523124, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672523124, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672606403, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7748522162437439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672606404, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672606404, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672621685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145123615860939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672637198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12009026110172272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672652705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.114539735019207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672668236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11232121288776398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672683736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11119571328163147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672698952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12166478484869003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672714460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225560307502747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672729376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928095132112503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672745393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097585409879684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672760906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902762621641159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672776312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022423207759857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672792308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098194345831871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672808202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488779097795486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672823830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127091944217682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672823830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672823831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672908186, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749757766723633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672908187, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672908187, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672923931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11357761174440384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672939771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12933164834976196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672955277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861459374427795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672970414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128029078245163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672985344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406569182872772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673000654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128710955381393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673015984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11429277807474136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673031489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405420303344727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673046704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632790625095367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673061922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871008783578873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673077403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12913395464420319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673092654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328411102294922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673108455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132471114397049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673124211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656660586595535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673124212, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673124212, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673209086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752427458763123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673209087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673209087, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784673224349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09740118682384491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673240022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206184327602386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673255530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227216050028801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673271106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1065693199634552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673286605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012135446071625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673302052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804464668035507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673317284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125197634100914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673332294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123235747218132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673347665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348530113697052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673362497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11794713884592056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673377400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846291482448578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673392599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11153062433004379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673407934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0931161642074585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673422911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042311042547226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673422912, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673422912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673505961, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752667665481567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673505961, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673505962, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673521075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274146497249603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673536039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09702027589082718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673550853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529821902513504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673565669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854652523994446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673580437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975764691829681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673595518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441706329584122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673610448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10833382606506348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673625627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11400409787893295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673640643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001153290271759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673655928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661642462015152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673670836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09791809320449829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673685694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09755630791187286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673700468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288963466882706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673715469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10149537771940231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673715470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673715470, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673798467, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752794623374939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673798468, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673798468, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673813359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354232043027878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673828526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353801399469376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673843681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430397093296051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673858862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755624622106552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673874022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299398750066757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673889346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100792795419693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673904814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013209074735641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673919915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568073391914368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673935111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723371058702469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673950164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602562874555588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673965461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10481980443000793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673980284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089240163564682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673995055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621747374534607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674010414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1137860044836998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674010468, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674010468, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674094435, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754996418952942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674094436, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674094437, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784674109294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085377112030983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674124417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214891821146011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674139767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767322778701782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674155077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777486115694046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674170079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10060925781726837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674185383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285207629203796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674200646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09631936252117157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674215509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877624899148941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674230239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017919540405273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674245062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278210043907166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674260222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10193649679422379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674274962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344456881284714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674289698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997213631868362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674304572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426849126815796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674304624, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674304625, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674390085, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754709720611572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674390086, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674390086, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674405076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09839719533920288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674420006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760672390460968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674435144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998426228761673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674449998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402244329452515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674464781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09421024471521378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674480076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710486769676208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674495470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435770452022552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674510670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10422546416521072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674526096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530197620391846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674541235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204942315816879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674556676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09690681099891663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674572152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11176567524671555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674587518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242222040891647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1019306480884552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602554, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674602555, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674688179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756123542785645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674688180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674688180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674703140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09839025884866714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674718428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752486437559128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674733554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10198871046304703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674748141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069134920835495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674763240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089421272277832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674778101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10909280180931091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674793002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080993920564651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674808257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09956123679876328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674823395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931626707315445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674838664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991813987493515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674853658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061269253492355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674868699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09499377757310867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674883602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677860885858536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674898745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632886737585068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674898746, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674898746, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674984086, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758821249008179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674984087, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674984088, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674999186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338009893894196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675014323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420866847038269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675029510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152126848697662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675044546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572750866413116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675059440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149033159017563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675074369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096921771764755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675089192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990338772535324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675103830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10203257203102112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675118565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790542513132095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675133216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10258839279413223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675148080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09866001456975937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675163477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242654174566269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675178609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11975989490747452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675193708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986080765724182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675193709, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675193710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675277383, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7763038277626038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675277384, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675277384, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784675292597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11662010103464127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675307518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126511752605438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675322925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003381535410881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675338114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10146120935678482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675353212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676675289869308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675368259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003376752138138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675383271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083489120006561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675398396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677052289247513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675413044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619533807039261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675427556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09495513141155243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675443116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488119721412659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675458171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210099816322327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675473417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09441406279802322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675488802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042594373226166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675488803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675488803, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675571657, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7762972712516785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675571657, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675571658, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675586884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11140605062246323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675602198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401738226413727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675617248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005912497639656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675632325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774566233158112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675647203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10376131534576416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675661790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1041513979434967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675676452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662417113780975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675691425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087929755449295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675706265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903056919574738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675720816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812669456005096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675735641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242415010929108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675758683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985153391957283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675773728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283553600311279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675790980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032082736492157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675790981, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675790981, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675873549, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766724824905396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675873550, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675873550, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675888652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.102554552257061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675903398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868321359157562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675918067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250212043523788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675933410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09488393366336823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675948885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475891083478928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675964122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797204077243805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675979297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11639493703842163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675994292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10128704458475113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676009565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057250127196312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676025274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11094112694263458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676040509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819766879081726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676055688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10858774185180664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676070941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09876522421836853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676085784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026102602481842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676085785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676085786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676166819, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766553163528442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676166820, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676166820, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784676182492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029969155788422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676197480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494790226221085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676212259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09250514209270477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676227895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006196066737175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676243220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905735939741135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676258589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1110759899020195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676274384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0963239073753357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676289933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049552857875824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676305418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655958205461502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676320617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10007711499929428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676335888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483012348413467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676351069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516394674777985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676366236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11157233268022537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676381627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552007496356964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676381628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676381629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676462689, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764861583709717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676462690, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676462690, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676477758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011183261871338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676493180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507632046937943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676508232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10512138903141022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676523514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890284925699234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676538484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251019150018692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676553781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976414829492569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676568893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386304557323456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676583858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260077357292175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676598992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133678466081619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676614107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09619663655757904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676629083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542993992567062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676644418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253558099269867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676659796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255686193704605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676674785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11880583316087723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676674785, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676674786, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676758965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7769604921340942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676758966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676758966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676774259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10123417526483536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676789159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811395406723022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676804323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602284967899323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676819710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242923885583878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676835110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11838633567094803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676850318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09676345437765121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676865605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574407875537872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676880486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201616585254669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676895891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206317156553268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676911137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10460135340690613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676925731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665298253297806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676941450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991105437278748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676956715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002788320183754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676972415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607842892408371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676972415, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676972416, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784677054888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771864533424377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784677054889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784677054889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784677070427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255187541246414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677086159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174723505973816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677101749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698425769805908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677117533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214948520064354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677133036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11576981842517853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677148731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312933474779129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677164434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004241347312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677179970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442195624113083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677195724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10565286874771118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677211523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777423530817032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677227470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727937519550323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677243176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690651088953018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677259139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776656866073608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677274563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11611852049827576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677274564, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677274565, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677356761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7778541445732117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677356761, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677356762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677372603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269893497228622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677388181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534816235303879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677404000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322686284780502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677419908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11689040064811707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677435730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10927785933017731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677451455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664503276348114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677467342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614991188049316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677483131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420412361621857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677499370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184493824839592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677515217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584299266338348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677530835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182144358754158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677546575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11299356073141098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677561865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840726643800735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677577653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11581984907388687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677577654, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677577654, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677659945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777467966079712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677659946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677659946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677675179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013793870806694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677690728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114591360092163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677706069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519380658864975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677721446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10206017643213272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677737044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061786323785782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677752718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903353989124298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677768129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0988330990076065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677783310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721451044082642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677798526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208886861801147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677813589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574379563331604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677828880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494816839694977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677844080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111842468380928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677859159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11758257448673248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677874500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378920078277588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677874501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677874501, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677956197, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780810594558716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677956198, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677956198, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677971462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359893530607224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677986943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769675672054291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678002152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175380647182465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678017515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731504648923874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678032733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11247068643569946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678048204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11861444264650345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678063785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027984529733658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678079140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827679187059402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678094682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10870544612407684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678110201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099226176738739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678125792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1196284368634224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678141279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341071128845215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678156781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053113266825676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678172490, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11271516978740692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678172491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678172491, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678256449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7783223986625671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678256451, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678256451, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784678271383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506192594766617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678286690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106657981872559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678301686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11880674213171005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678317204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732235014438629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678331979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612025111913681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678346983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194278508424759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678362694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703060030937195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678377671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085426807403564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678392827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11422624439001083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678407786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09439211338758469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678422949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11763293296098709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678438052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145939886569977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678452754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09916466474533081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678467509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867501050233841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678467510, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678467510, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678551123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777716517448425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678551124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678551124, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678566057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10353630781173706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678581367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944032371044159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678596573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09722398221492767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678611599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829444974660873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678626752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10959458351135254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678641865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11292397230863571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678656849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678759425878525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678672014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067994087934494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678687022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10726537555456161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678701948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145429313182831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678716895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960593283176422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678732330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10030815005302429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678747496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11159928143024445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678762359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985998809337616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678762360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678762361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678845616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7781968712806702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678845617, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678845617, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678860864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628467798233032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678875834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09645366668701172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678891016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542475432157516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678906386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09636390209197998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678921953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329315811395645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678937304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270583420991898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678952702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1081763282418251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678967622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09610337764024734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678982663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605404317378998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678997769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436414927244186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679012501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10332357883453369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679027985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197797417640686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679042703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278233885765076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679057681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021430566906929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679057681, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679057682, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679142129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784578800201416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679142130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679142130, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784679157321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10187067091464996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679172524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984029084444046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679187804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311678797006607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679202682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028367206454277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679218142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09907148778438568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679233202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10262982547283173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679247987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09637533128261566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679262821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991363674402237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679278111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10390663892030716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679293203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911276191473007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679308208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480337589979172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679323268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770425200462341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679338552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260921716690063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679353437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09656118601560593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679353438, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679353438, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679437449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784078121185303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679437450, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679437450, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679452765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09580580145120621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679467600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023661702871323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679482554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785361588001251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679497221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09987707436084747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679512157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433676838874817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679527776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09610987454652786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679542724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10323725640773773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679557945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09868337959051132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679572922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983649417757988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679588147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577677935361862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679603447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0930241197347641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679619010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821833461523056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679634395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570856928825378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679650022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046188324689865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679650023, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679650023, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679732921, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7787220478057861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679732922, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679732922, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679748156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055678129196167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679763546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10207225382328033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679778802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10617764294147491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679794532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363797098398209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679809440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09720319509506226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679824397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10495387017726898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679839818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961461290717125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679854837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11460188031196594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679869896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09856100380420685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679885457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0908965989947319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679900858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10059625655412674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679916051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10558567941188812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679931194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10363882035017014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679946363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10908914357423782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679946364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679946364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784680030020, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789427638053894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784680030021, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784680030021, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784680045355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184445977210999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680060244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658824443817139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680075167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809127449989319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680090233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10048490017652512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680105519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089654475450516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680120393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09549888968467712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680135383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453785955905914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680150562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034712865948677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680165334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090667754411697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680180494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044096946716309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680195181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823154658079147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680210398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107409954071045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680225521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064695030450821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed681082816.log b/recommendation_v4/rcp_logs/gbs_8192/seed681082816.log new file mode 100644 index 000000000..f49611129 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed681082816.log @@ -0,0 +1,1916 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651299073, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651299074, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651319338, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "seed", "value": 681082816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651319340, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651319341, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651319341, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651319341, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784651990338, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784651990341, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784651990655, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652116205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771222531795502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652128040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14031238853931427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652140285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13862194120883942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652152734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13962367177009583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652165410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139175146818161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652178015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888496160507202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652190594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970795273780823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652203332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822603225708008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652216107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13930273056030273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652228841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13943462073802948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652241690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13887378573417664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652255483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867473602294922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652268607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13935598731040955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652282910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137485072016716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652282911, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652282912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652363758, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49788910150527954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652363759, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652363759, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652376241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938696682453156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652388939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13919930160045624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652402121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13911646604537964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652415602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13970261812210083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652428755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395055055618286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652441755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13960090279579163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652454311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395157128572464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652467169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395343840122223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652480530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395704597234726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652493719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391899734735489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652506942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1389874517917633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652520079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13899360597133636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652533063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13915914297103882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652546443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13897249102592468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652546443, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652546444, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652627704, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.498083233833313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652627705, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652627705, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652641039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905125856399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652654441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898025453090668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652667796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1391158550977707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652680962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393146514892578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652693933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388390213251114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652706889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885082304477692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652719908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880310952663422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652733421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1390085071325302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652746693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138760045170784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652759910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13847298920154572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652773312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863687217235565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652786342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13854920864105225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652799903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385686844587326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652813059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870517909526825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652813060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652813060, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652895118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49833664298057556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652895120, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652895120, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784652908122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382170468568802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652921491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828502595424652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652934749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818231225013733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652948460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822489976882935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652961711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834209740161896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652975057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845370709896088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652988358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380363553762436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653001585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818314671516418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653015083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13778428733348846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653028615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871639966964722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653041991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768532872200012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653055310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836461305618286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653068833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690422475337982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653082339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776034116744995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653082339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653082340, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653163345, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49876242876052856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653163347, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653163347, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653176359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759009540081024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653189464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698963820934296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653202421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379309743642807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653215767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364665925502777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653229204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374896764755249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653242483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583959639072418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653255704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368633210659027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653268830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656987249851227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653281888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13803702592849731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653294790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13655391335487366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653307752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14393864572048187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653321078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652613759040833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653334047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569329679012299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653347051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379922777414322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653347052, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653347052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653428831, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4993664026260376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653428832, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653428833, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653441892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362362802028656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653455055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13900673389434814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653468298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366998255252838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653481324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347973644733429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653494531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358330249786377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653507980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875670731067657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653521297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13481369614601135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653534525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13460899889469147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653547728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471469283103943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653560525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334437131881714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653573541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312631845474243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653586293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13595062494277954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653599321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13368016481399536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653612559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846777379512787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653612560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653612560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653696254, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5001416802406311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653696255, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653696255, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784653709402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134684219956398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653722468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13439500331878662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653735650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13626544177532196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653748903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355787217617035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653762137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735981285572052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653775429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13007204234600067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653788474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356765627861023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653801711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098154962062836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653815248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431890308856964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653829116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336804509162903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653842518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13411881029605865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653856009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486875593662262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653869968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564422726631165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653883541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13445627689361572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653883542, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653883542, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653967756, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5012380480766296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653967757, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653967757, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784653981430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326255977153778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653994952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433200120925903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654008532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13558852672576904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654022469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550186157226562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654036189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869377970695496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654049782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361084461212158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654063885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532856106758118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654077342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367904722690582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654091029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336812824010849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654104949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244874775409698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654118501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13586358726024628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654132419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396228849887848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654146099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13365484774112701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654159655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107936084270477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654159656, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654159656, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654243693, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5025910139083862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654243694, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654243694, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654257888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757383823394775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654271850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496515154838562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654285803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533110916614532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654299848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166607916355133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654313495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383617997169495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654327403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433513045310974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654341382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13675826787948608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654355279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13413068652153015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654368988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312189280986786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654382362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299944967031479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654395430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12807582318782806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654408690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132929265499115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654422093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988968193531036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654435065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1253114640712738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654435065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654435066, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654516140, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5040611028671265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654516141, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654516141, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654529569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131403848528862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654542748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13582263886928558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654556129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337825238704681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654569546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130586177110672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654582653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12853679060935974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654595983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126909077167511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654609823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924300134181976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654623193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13222657144069672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654636603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544265925884247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654650044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12535491585731506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654663281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881585955619812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654676491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1341819167137146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654690050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104036450386047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654703910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948203086853027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654703910, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654703911, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654784707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5059241652488708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654784708, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654784708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784654798528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13038645684719086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654812020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1293793022632599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654825211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296723365783691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654838900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1241694912314415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654852703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12957152724266052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654865991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.125825434923172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654879192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301743984222412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654892606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12203685194253922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654905734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358071118593216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654919381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13708838820457458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654932757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1237095519900322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654946436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12345212697982788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654960004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256718933582306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654973505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12905652821063995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654973505, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784654973505, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655057144, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5080748796463013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655057145, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655057145, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655070370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645760715007782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655084202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364504098892212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655097812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356598138809204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655111315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13995501399040222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655124892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654816150665283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655138548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13822610676288605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655152158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12747043371200562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655165792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261124163866043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655179357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227652549743652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655193433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411198377609253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655207423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127297043800354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655221182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282861828804016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655234875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13270065188407898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655248085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317131519317627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655248085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655248086, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655333023, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5103504657745361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655333025, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655333025, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655346993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14135098457336426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655360970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14247478544712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655374616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530953228473663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655388430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13945241272449493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655402436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14491033554077148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655416180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14077487587928772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655430106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583334743976593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655444049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752244412899017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655458125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13185761868953705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655472140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602018356323242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655486090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788817822933197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655499783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329811066389084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655514266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458189368247986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655528755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13663744926452637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655528756, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655528756, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655609965, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5127341747283936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655609966, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655609966, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784655623869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872632384300232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655638121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381729692220688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655652128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13167187571525574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655666205, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295377790927887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655680317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376925885677338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655694649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13961435854434967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655708835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13256706297397614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655723236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411445587873459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655736910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14070191979408264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655750896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13514721393585205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655764653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12765295803546906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655778225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988077104091644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655791763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416153192520142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655805445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12771421670913696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655805445, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655805446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655884246, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5158132314682007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655884247, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655884247, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784655897823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132221519947052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655911418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687501847743988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655925203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376812636852264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655938923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321364790201187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655952786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13392853736877441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655966470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334347426891327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655980520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372154533863068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655994489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282378017902374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656008169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13983415067195892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656022187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319025307893753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656035861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12884311378002167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656049281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311187446117401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656062779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216355443000793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656076519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13587895035743713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656076519, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656076520, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656156669, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5200971961021423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656156671, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656156671, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656170305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332913339138031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656183793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12287377566099167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656197258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305890828371048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656210866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306329071521759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656224468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512910902500153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656238713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532572984695435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656252462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529640436172485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656266760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12388919293880463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656280803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308993101119995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656294397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122868537902832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656308260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023260235786438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656321824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303622603416443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656335479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091444969177246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656348856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13019375503063202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656348857, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656348858, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656429404, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5258970856666565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656429407, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656429407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656443044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777141273021698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656456731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228681683540344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656470324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334223747253418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656483872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044852018356323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656497434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268361508846283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656510899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324801743030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656524808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356820911169052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656538656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327984780073166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656552463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311853677034378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656566355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13906383514404297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656580175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375268876552582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656593493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13757941126823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656607712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181786239147186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332755982875824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656621487, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656621487, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656700406, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5335649251937866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656700406, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656700407, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784656714012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116003572940826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656727586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496986031532288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656741235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376531720161438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656755057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331360936164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656768780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12733836472034454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656782524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12920856475830078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656796314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12585532665252686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656809946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12394177913665771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656823991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600575923919678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656837996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716623187065125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656851659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1345822811126709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656866077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12673257291316986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656880071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305486261844635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656893494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12341387569904327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656893494, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656893495, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656973926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.544227659702301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656973927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656973927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784656987789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13216707110404968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657001416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263165950775146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657015007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13363540172576904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657028521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376463919878006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657042159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360497772693634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657055757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114242255687714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657069280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12700693309307098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657082877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134910449385643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657096332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12443897128105164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657110040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300889402627945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657123722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235530257225037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657137429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13375939428806305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657151177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14009296894073486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657164675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594098389148712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657164676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657164676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657246803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5590075254440308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657246804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657246804, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657260834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363396942615509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657274447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313738077878952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657287979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237138092517853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657301671, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13870389759540558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657316103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298091560602188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657330593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382077932357788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657344635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13470271229743958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657358860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13714686036109924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657372908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433098793029785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657387101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029754161834717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657401310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14035071432590485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657415523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344282478094101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657429624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12867119908332825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657443873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322600722312927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657443873, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657443873, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657527143, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5742246508598328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657527144, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657527144, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784657541585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14148971438407898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657555821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13248011469841003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657570524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13337033987045288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657584760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023550808429718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657599394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349516659975052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657613675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321468234062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657627793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334597647190094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657641979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13504600524902344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657655956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13060791790485382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657670035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13214240968227386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657684115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264547288417816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657698177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366458535194397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657711967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593672215938568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657726248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703280687332153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657726248, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657726249, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657807018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5902137160301208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657807018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657807019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784657821020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129399716854095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657835262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506154716014862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657849228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378238677978516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657863611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13147683441638947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657877776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13669532537460327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657891744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12872272729873657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657905910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323192298412323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657919718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12303484976291656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657933586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13013719022274017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657947394, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13073085248470306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657961196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303349733352661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657974894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1305142045021057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657988683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12807978689670563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658002444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13004754483699799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658002445, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658002445, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658084872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6024594306945801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658084873, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658084873, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658098464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158553302288055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658112320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259918510913849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658126028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12806397676467896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658139890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13228440284729004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658154102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308874487876892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658167968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12072835117578506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658182211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289505958557129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658196255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311330497264862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658210314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13529877364635468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658223925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699222564697266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658238385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166925311088562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658252573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11702980846166611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658266958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11962512135505676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658281447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12414347380399704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658281447, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658281448, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658364315, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6141406297683716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658364316, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658364316, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658378640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12031309306621552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658392771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1266314536333084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658407152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268780529499054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658421326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299523413181305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658435477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11986198276281357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658449661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330856829881668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658463664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278616189956665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658477845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11958590894937515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658492104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12528091669082642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658506701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12477200478315353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658521198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11660575866699219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658535963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13023051619529724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658550433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12538494169712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658564773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12739627063274384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658564774, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658564774, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658646993, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.625512957572937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658646994, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658646994, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784658660764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12582017481327057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658675259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11704692244529724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658689090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12046000361442566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658703060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11908119916915894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658716830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12203685194253922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658730629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12434214353561401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658744345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11421803385019302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658758044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12087755650281906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658771709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12316860258579254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658785262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162988632917404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658798765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11407345533370972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658812475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12167395651340485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658825988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727016627788544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658839809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1211378201842308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658839809, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658839810, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658919591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6336947083473206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658919592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658919592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784658933665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11395083367824554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658947676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114786565303802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658961759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12506119906902313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658975977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11942973732948303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658989830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12177443504333496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659003634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131211370229721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659017142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674564331769943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659031416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12032938748598099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659045491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11837728321552277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659059502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018408834934235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659073810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11828451603651047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659087843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195090040564537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659101809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12300071865320206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659116198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11555109173059464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659116199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659116200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659196400, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6453458666801453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659196401, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659196401, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659210604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778213083744049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659224665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088983565568924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659238628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818643122911453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659252904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11244756728410721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659266740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765799880027771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659280897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11969335377216339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659295136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131792888045311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659309678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11826705932617188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659323795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10036175698041916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659338445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12030018121004105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659352859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126769408583641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659367200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214502155780792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659381301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801862925291061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659395363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11692067980766296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659395364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659395364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659478469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6653547286987305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659478470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659478471, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784659492308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1198120042681694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659506316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11910925805568695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659520518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12182479351758957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659534503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448430478572845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659548604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235140264034271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659562479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11653943359851837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659576652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11874068528413773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659590610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832110583782196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659604547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699468106031418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659618386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11877104640007019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659632215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320126801729202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659645817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1157936379313469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659659461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228439956903458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659673547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553605437278748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659673548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659673548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659757933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6942802667617798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659757933, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659757934, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784659772123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239108219742775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659786347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669708043336868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659800480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601518839597702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659815071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12078433483839035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659829482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823052376508713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659843883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103881150484085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659857834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225128918886185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659872006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084471195936203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659886040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104283332824707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659900540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521189659833908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659914752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12328799813985825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659928986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524202674627304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659943421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723641097545624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659957772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11291384696960449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659957773, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784659957773, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660039846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7334295511245728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660039847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660039847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660054279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257617026567459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660068611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334279924631119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660082828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147313341498375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660097474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238206177949905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660111869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11894986778497696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660126484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11801910400390625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660141330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12169331312179565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660156006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12067268788814545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660170455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387570947408676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660185536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202207207679749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660200173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11507386714220047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660214597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376737058162689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660228945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256818026304245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660243628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156059131026268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660243629, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660243629, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660324715, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7474913001060486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660324716, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660324716, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660339130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1218007504940033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660353734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185261607170105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660367987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328695714473724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660382551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454910039901733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660396777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802781581878662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660411341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11497291922569275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660425649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841320991516113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660439638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123886704444885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660453915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033175140619278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660468057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080699935555458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660482649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12190952897071838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660497327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547273606061935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660511799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12042888253927231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660526760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12023081630468369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660526760, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660526760, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660610335, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7542760372161865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660610336, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660610336, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784660624898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1185731291770935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660639246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757932603359222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660653752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162232980132103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660668304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550933122634888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660682549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328785330057144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660696821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289867967367172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660711029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1186278760433197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660725113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144903227686882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660739158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407258570194244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660753204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10236520320177078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660766962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10750637203454971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660781066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11270975321531296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660794995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149738729000092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660808988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11458010226488113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660809032, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660809032, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660890204, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7592202425003052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660890205, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660890205, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784660904283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486529767513275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660918569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656307637691498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660932903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427957564592361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660946909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11000615358352661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660961179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694421827793121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660975600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503966152667999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660989998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311264336109161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661004152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11152707785367966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661018255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11597553640604019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661032705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11030151695013046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661046914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09860848635435104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661061315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11305255442857742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661075434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752210021018982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661089721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050916314125061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661089722, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661089722, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661170707, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7619774341583252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661170707, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661170708, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661184831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658600181341171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661199034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278734356164932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661213173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09554534405469894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661227319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350877791643143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661241468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240158975124359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661255517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10520248115062714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661269751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08772898465394974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661284807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.109085813164711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661298947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101845115423203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661313186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772070080041885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661327485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781719535589218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661341302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961709380149841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661355724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09852872788906097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661370060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679809749126434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661370060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661370062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661451269, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7638276219367981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661451270, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661451270, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784661465651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11501955986022949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661479928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042746156454086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661493578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336939245462418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661507882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10514426976442337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661521944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09917531162500381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661536294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10130565613508224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661550603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1062275841832161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661564936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10616952180862427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661579174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09760165959596634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661593731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836067795753479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661608058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391271859407425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661622300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671212524175644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661636561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10830670595169067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661650864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267375946044922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661650864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661650864, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661735179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7647554278373718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661735180, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661735180, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784661749583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10994894057512283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661763574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845544397830963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661777799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259705036878586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661792089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507544875144958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661806516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057591438293457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661820985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309661716222763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661835309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046101361513138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661849702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499198734760284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661864140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836893320083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661878276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511566698551178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661892604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263691633939743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661906492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09397896379232407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661920936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784488379955292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661935002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682113468647003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661935003, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784661935003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662017544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7661838531494141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662017545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662017545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662031456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09116104245185852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662045564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10436365753412247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662059600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025796830654144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662073907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09783252328634262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662087919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11126011610031128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662102088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118263006210327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662116208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09761209785938263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662130115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11190624535083771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662144197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507293045520782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662158224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448917746543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662172474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072661355137825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662186385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042269766330719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662200335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09774293005466461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662214584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643064230680466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662214585, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662214585, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662296360, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7672280669212341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662296361, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662296361, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662310922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035892590880394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662325452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398008674383163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662339570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10613439232110977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662353909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365023463964462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662368367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09767122566699982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662382635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09935953468084335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662397142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643425583839417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662411044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09614503383636475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662425389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790236294269562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662439635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086350828409195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662453769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1163138672709465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662468001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387752741575241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662482019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638100445270538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662496360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538861691951752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662496360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662496361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662577634, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7676960825920105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662577635, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662577635, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784662591566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032930225133896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662605574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11560408771038055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662619294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793529450893402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662633349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10197356343269348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662647210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09172068536281586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662661266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863091260194778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662675049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431773215532303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662689199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743609070777893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662703092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765986889600754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662716974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071826726198196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662731318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1127861738204956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662745669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09931215643882751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662760066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097998395562172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11187499761581421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662774046, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662774046, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662854482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7682220339775085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662854483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662854483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784662868999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757115483283997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662883397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151541233062744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662897675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11104463785886765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662912193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09136421233415604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662926178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11455836892127991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662940143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672984272241592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662955209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110944002866745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662970081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118093878030777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662984966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12281898409128189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662999772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536635667085648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663014500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910400748252869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663029277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12319110333919525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663043919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09918997436761856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056973040103912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663058547, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663058548, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663137837, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7681711912155151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663137838, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663137838, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663152432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391851514577866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663167180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11840350925922394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663181860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11008424311876297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663196772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10294250398874283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663211535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09677263349294662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663226326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11262580752372742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663241465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652157664299011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663256398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10488476604223251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663271437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11651985347270966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663286178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996176227927208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663301262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12480071187019348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663316325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769788175821304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663331156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442700773477554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663345834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731793940067291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663345835, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663345835, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663427060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7683420181274414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663427062, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663427062, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784663441905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338063329458237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663456485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600467562675476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663471051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156890541315079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663485937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11822232604026794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663500462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435253918170929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663515091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11838249117136002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663529853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127475306391716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663544249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12065243721008301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663558682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160629466176033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663573437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046437174081802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663587853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10813063383102417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663602549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762616991996765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663616936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11463689059019089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663631611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11375244706869125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663631611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663631612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663713301, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7691826224327087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663713302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663713302, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784663728115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11989345401525497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663742786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723431408405304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663757155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781346261501312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663772004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841161012649536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663786746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112009733915329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663801419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864001512527466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663816023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370762437582016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663830564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880890488624573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663845159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476534068584442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663859626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006606578826904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663874351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439606010913849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663888859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606295078992844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663903258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040777862071991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663917668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090184956789017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663917668, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784663917669, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784663998901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7701598405838013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784663998902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784663998902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664013133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991422086954117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664027380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006740853190422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664041352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09688588231801987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664055238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810989141464233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664069300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474083364009857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664083316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10493630915880203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664097237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491269826889038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664111243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462367534637451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664125224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194014012813568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664139257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.120423324406147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664153480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11735742539167404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664167645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154299437999725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664182001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10065121948719025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664195944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12107671797275543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664195944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664195945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664277077, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7704293131828308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664277078, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664277078, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664291507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095532774925232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664305700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09657801687717438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664319863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479463636875153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664334067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370733588933945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664348145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795273631811142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664362382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529375821352005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664376754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10605141520500183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664391023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11178549379110336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664405402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502991080284119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664419860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774807631969452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664434130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10560641437768936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664448410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11427101492881775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664462302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694784671068192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664476306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021760106086731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664476306, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664476307, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664557184, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712799906730652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664557185, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664557185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784664571300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0991738885641098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664585594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09363225102424622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664599743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0962313860654831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664614017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09689575433731079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664628161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671168774366379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664642576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09427580237388611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664656762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10013239830732346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664670837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972953736782074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664685198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295893996953964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664699325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865352302789688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664713780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449165105819702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664727990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030883640050888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664742320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396458745002747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664757255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114034056663513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664757255, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664757255, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664840316, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713361382484436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664840317, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664840317, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784664855008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09833411872386932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664869550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716298222541809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664884346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10992131382226944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664898797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148195177316666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664913108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978289902210236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664927660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074284166097641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664941775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034887284040451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664956069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10621485114097595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664970601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247448086738586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664984683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079622268676758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664998794, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999419167637825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665012948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230159014463425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665026970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10905510187149048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665040828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11467419564723969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665040829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665040829, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665121201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7716841101646423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665121202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665121202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665135253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10946009308099747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665149352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666421055793762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665163349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09102733433246613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665177262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151714831590652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665191469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597078502178192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665205309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077072024345398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665219200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985107719898224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665233375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264549404382706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665247710, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09986155480146408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665262081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10374214500188828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665276659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273449122905731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665290999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747628659009933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665305571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467890650033951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665320182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10982658714056015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665320182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665320183, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665399759, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720115184783936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665399760, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665399760, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784665414121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321003943681717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665428435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10273704677820206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665442533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10391547530889511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665456389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214047133922577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665471188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672449320554733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665485445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068007558584213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665499669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809403866529465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665514228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10180290788412094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665528658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11123138666152954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665543171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484782606363297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665557534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10367661714553833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665571840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10224922001361847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665586480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10311344265937805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665600807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697071254253387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665600808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665600808, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665680079, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7725686430931091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665680080, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665680080, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784665694457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764146596193314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665708902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158310830593109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665722972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400755703449249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665737197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131391882896423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665751565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511216521263123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665765522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11246787756681442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665779859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104936361312866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665793896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021754890680313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665808379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11242365092039108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665823066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687392950057983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665837378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10317914187908173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665851635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506613552570343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665866282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10481679439544678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665880763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817253798246384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665880763, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665880764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665960225, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731200456619263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665960225, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665960226, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784665974501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094622522592545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665988829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0993785709142685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666003022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058211550116539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666016779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007244810461998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666031502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729334503412247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666045917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10922595858573914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666060718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751581192016602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666075660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11521542817354202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666090318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377851665019989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666104884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992086887359619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666119355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395169258117676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666133762, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299262404441833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666148261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521083325147629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666162621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728824138641357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666162622, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666162622, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666241629, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733516693115234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666241630, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666241630, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666256235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121877491474152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666270717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038679093122482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666285207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865560173988342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666299476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476656258106232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666313690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09417548030614853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666328334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215829104185104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666342444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10109510272741318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666356626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764233767986298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666371102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304410010576248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666385590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292252898216248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666399725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971666872501373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666414159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430055648088455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666428679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567770898342133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666443064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248652845621109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666443065, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666443065, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666522970, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7732779383659363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666522971, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666522971, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784666537899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11289908736944199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666552399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10876475274562836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666566816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931092500686646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666581522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687839984893799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666595781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137956589460373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666610049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10763005167245865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666625044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832745373249054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666639679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899398475885391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666654393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11562903225421906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666669422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12212959676980972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666684150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071389690041542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666699022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10978469252586365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666713976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482860893011093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666728691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022135615348816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666728692, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666728692, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666807623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773613691329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666807624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666807624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784666822293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314789950847626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666837248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958286374807358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666852129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034375429153442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666867089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10665416717529297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666881881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428523272275925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666896739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369027197360992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666911429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10663395375013351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666926103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457222700119019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666941001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278854310512543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666955848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12005262076854706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666970853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105778269469738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666985579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727311462163925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667000376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10682874917984009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667015513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11832724511623383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667015514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667015514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667095862, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7736829519271851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667095863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667095863, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667110772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11913055926561356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667125531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805072635412216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667140326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1195872575044632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667154808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11377354711294174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667169382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747969150543213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667184244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11543317884206772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667198798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095930635929108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667213606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11623881757259369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667228524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166904121637344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667243325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573405027389526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667258013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234522610902786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667272831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645903646945953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667287359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657964438199997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667302118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491914302110672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667302118, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667302118, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667384936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746788263320923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667384937, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667384937, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784667399508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10979031026363373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667414403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11897087097167969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667429017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11021392047405243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667443913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352507770061493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667458668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545757204294205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667473316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911735892295837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667488204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820864140987396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667503281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10950326919555664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667518360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237206310033798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667533178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174982041120529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667548259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991306602954865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667563206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059160530567169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667578124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170601099729538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667592919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904745012521744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667592919, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667592920, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667675497, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746866345405579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667675498, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667675498, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784667690161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11775237321853638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667704686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052800714969635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667719124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099445819854736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667733500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10292735695838928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667747916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547758638858795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667762462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163219809532166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667776730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10764408856630325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667790838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749578475952148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667805065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09757620841264725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667819250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1016073152422905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667833582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584770888090134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667847595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468987375497818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667861922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014806479215622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667876380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11553658545017242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667876381, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667876381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667958018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749338746070862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667958019, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667958019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784667972235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449310183525085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667986592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529280453920364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668000828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10047170519828796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668015236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09215429425239563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668029504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405976116657257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668043586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10259072482585907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668058104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218916088342667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668072910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219366639852524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668087427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728734731674194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668102252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301481932401657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668116981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10890769213438034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668131841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001032590866089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668146787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072750836610794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668161425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10690552741289139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668161425, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668161426, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668242772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7753920555114746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668242773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668242773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784668257162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897976905107498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668271689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340233892202377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668285895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074934154748917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668299966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0988878682255745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668314684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603939533233643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668329012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056118458509445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668343195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10334772616624832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668357849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10828372091054916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668372333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108609639108181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668386346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853592306375504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668400523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099872887134552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668414828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887070208787918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668428820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10614613443613052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668442870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08660341799259186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668442871, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668442871, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668523653, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756896615028381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668523654, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668523654, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784668537890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09549378603696823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668552135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164761334657669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698570311069489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668580499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125316679477692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668594760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219017416238785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668609214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0903412327170372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668623393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298417508602142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668637588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09710589051246643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668651624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331118851900101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668665921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611691325902939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668680587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098966896533966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668695086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10785926878452301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668709417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625440627336502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668723822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10316698998212814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668723823, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668723824, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668807418, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759447693824768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668807419, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668807420, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784668821918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09548661857843399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668836700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518413782119751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668851206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0949997678399086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668865655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10181041806936264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668880140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09792274981737137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668894131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10052938014268875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668908084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928506404161453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668922604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10080427676439285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668936696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868903994560242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668951029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1091141477227211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668965718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633743554353714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668980323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10407593101263046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668994796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901889204978943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669008995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10821304470300674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669008996, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669008996, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669091639, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765141725540161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669091640, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669091640, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669105843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649342089891434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669120129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10388552397489548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669134522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10322399437427521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669148840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09971053153276443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669163146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695608705282211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669177429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09616955369710922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669192304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483883321285248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669206594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318107903003693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669220780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057581752538681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669235266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879183769226074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669249226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09726916253566742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669263277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10195410251617432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669277604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09050723165273666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651710629463196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292091, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669292091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669374807, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767730355262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669374808, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669374808, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784669389173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09895555675029755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669403693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10061222314834595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669417825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09943683445453644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669432460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912155359983444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669446835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028618574142456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669461379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238278657197952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669475606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10096438974142075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669489865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557282716035843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669504323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160498321056366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669518329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171219497919083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669532769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10114249587059021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669547103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778731107711792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669561333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10516048967838287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669575758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561524331569672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669575758, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669575759, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669657761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772966623306274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669657762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669657762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784669672150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551825910806656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669686399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09797800332307816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669700575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685323923826218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669715047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1060190200805664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669729403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10199534147977829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669743477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711319744586945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669757692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09762872755527496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669772248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977715253829956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669786607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483758896589279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669800766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382764786481857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669815092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214240103960037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669829408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996648669242859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669843888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634909570217133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669858257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995849072933197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669858258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669858258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669941550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776336669921875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669941552, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669941552, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784669955743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10237082839012146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669969890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527538508176804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669984317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09449677914381027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669998963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09801539778709412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670013295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10146176815032959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670028017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09982806444168091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670042298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067717969417572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670056873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996140539646149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670071510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834111273288727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670086191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09857259690761566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670101006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212171077728271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670115355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788954257965088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670130044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11204443871974945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670144303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042007803916931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670144303, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670144304, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670226572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7780176401138306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670226573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670226574, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784670240618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10549969226121902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670255117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10435344278812408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670269104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10606437176465988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670283412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1109551265835762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670297898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615909844636917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670312426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777010768651962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670326506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10738123953342438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670340707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09729626774787903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670354869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597806423902512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670368915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10325144976377487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670383107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772190243005753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670396967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036251038312912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670411342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608820617198944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670425425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09512599557638168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670425470, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670425471, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670505827, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779695987701416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670505828, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670505829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784670519966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09873918443918228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670533974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10496929287910461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670548092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260479897260666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670562376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453533381223679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670576498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10799968987703323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670590546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163819372653961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670604667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10816188156604767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670618878, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836572408676147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670633274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044707149267197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670647374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062265932559967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670661931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11493205279111862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670676424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785722196102142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670691132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638938099145889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670705602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351461172103882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670705650, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670705650, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670787037, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7784585952758789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670787038, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670787038, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784670801625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09608211368322372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670816135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09714801609516144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670830528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279899626970291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670845153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11728516966104507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670859507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11336110532283783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670873706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1105274185538292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670888484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694359242916107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670903113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1202782392501831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670917636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365551710128784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670932568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852912813425064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670947314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10975977033376694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670961799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1097491905093193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670976458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322633177042007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670991143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105816811323166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670991143, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784670991144, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671072744, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7789474725723267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671072745, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671072745, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671087471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705175250768661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671102060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980982333421707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671116927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414626985788345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671131654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013339459896088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671146382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10758160799741745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671161121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1076451987028122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671175966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120748728513718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671190784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118064820766449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671205505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11979889124631882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671220302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10937172919511795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671234932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866264998912811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671250042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033109799027443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671265199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10242220014333725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671280168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10883799940347672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671280168, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671280169, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671362379, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7792091965675354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671362380, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671362381, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784671377288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447247117757797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671392283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11856947094202042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671407181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325080692768097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671422229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11253128200769424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671437089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014106869697571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671451645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970248073339462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671466511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11164475232362747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671480876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10921857506036758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671495750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999234765768051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671510315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09839978069067001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671525097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099696159362793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671540259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10951043665409088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671555226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386184602975845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671570037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985489934682846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671570038, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671570038, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671651612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791255116462708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671651613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671651613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784671666437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124267727136612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671681531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273464560508728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671696657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825219005346298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671711673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175262928009033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671726527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258808523416519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671741674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782382637262344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671756800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437998712062836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671772120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302034556865692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671787126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630546510219574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671802063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829003155231476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671817284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12976817786693573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671832183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279603093862534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671847412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11167463660240173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671862536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610187590122223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671862537, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671862537, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671945064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794792056083679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671945065, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671945066, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784671959913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09501848369836807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671974869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069482564926147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671989725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12108268588781357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672004624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677757114171982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672019479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058828979730606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672034301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703586041927338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672049092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1123071014881134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672063744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338335275650024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672078420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303067415952682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672092777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778765916824341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672107321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10674039274454117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672122016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027026921510696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672136834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09202705323696136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672151107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10336937010288239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672151108, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672151109, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672233867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7794865369796753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672233868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672233868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784672248487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028187930583954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672262994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09613595902919769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672277283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136987954378128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672291640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696503520011902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672306011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967548936605453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672320759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10425501316785812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672335073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732364654541016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672349495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11266323179006577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672363869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09954506903886795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672378602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09508813917636871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672393099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09675423800945282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672407470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09635862708091736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672421560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269271582365036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672435914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014795511960983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672435914, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672435915, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672519971, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.780205249786377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672519972, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672519972, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784672534380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030035987496376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672549006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10239669680595398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672563449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338165611028671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672577800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664341598749161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672592443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230427980422974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672607245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09818974882364273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672622005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789645463228226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672636326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10540473461151123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672650645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623770952224731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672664712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10532168298959732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672679347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349265486001968, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672693557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648928582668304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672707597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530990362167358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672722405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279844492673874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672722450, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672722450, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672805703, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7805505990982056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672805704, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672805704, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784672820107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10832349210977554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672834709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11189933121204376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672849509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619846731424332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672864321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669924318790436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672878660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034600645303726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672893380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072776675224304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672908032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0955292358994484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672922391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626262426376343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672936690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09848978370428085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672951053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10117946565151215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672965718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085251927375793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672980004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240506380796432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672994399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09863270819187164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673008954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315186530351639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673009002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673009003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673090939, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7808977365493774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673090940, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673090940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673105526, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698577970266342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673120187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10713594406843185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673134614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836734086275101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673148966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139473527669907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673163194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09389389306306839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673177672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666466504335403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673192354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10248865187168121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673206737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024690493941307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673221577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.104263074696064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673236206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11129125207662582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673250998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09459499269723892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673265665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928961634635925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673280335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097629368305206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673294705, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090658813714981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673294753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673294753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673375867, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814043164253235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673375868, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673375868, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784673390218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09752576053142548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673404962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692720115184784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673419795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034731030464172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673434058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10506875813007355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673448686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10023622214794159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673462820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10820624977350235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673477068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10016319155693054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673491544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826312959194183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673505988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880913585424423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673520369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818726569414139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673534728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09867673367261887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673549254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09377811849117279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673563434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09401117265224457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673577791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10566861927509308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673577791, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673577792, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673661044, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819559574127197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673661045, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673661045, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784673675258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174286365509033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673689610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11322490870952606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673703961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949581325054169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673718270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386195033788681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673732662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10960400104522705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673746875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09980147331953049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673760996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093202456831932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673775148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101031512022018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673789316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673410445451736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673803474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002781018614769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673817827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09667924791574478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673832461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111208125948906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673847003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11825482547283173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673861418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082550659775734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673861418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673861419, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673943680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7817984223365784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673943681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673943681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784673958211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394734680652618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673972515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092375427484512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673987250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09976836293935776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674001622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028862953186035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674016060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482536256313324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674030463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09903731942176819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674044717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10026056319475174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674059378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053149551153183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674073661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10534701496362686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674087765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09344318509101868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674102568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288374871015549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674116899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10054522007703781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674131337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09389414638280869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674145982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981614351272583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674145982, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674145983, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674229884, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819409370422363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674229886, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674229886, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784674244474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11012573540210724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674259024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340060085058212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674273377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961215406656265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674287826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1070491224527359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674302158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223103314638138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674316459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10032615810632706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674330768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632577538490295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674345089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10818643867969513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674359546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09681913256645203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674374005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09644201397895813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674388397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1104249358177185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674402545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09639586508274078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674417123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10177382826805115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674431353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023176908493042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674431353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674431354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674514907, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7823194861412048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674514908, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674514908, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784674529349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1013331338763237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674543711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10639609396457672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674557961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116646230220795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674572773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09260784089565277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674587415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154565423727036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674602115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653949528932571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674616702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590153723955154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674631075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118746012449265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674645769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553213953971863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674660640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844563692808151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674675382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09607014805078506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674689958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642936080694199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674704815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777819365262985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674719210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300293564796448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674719211, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674719211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674802974, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825934886932373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674802975, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674802975, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784674817984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016173660755157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674832401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348278284072876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674846485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09196615219116211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674861206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082165896892548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674875618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733262449502945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674890058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075976490974426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674905072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961211770772934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674919733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034349650144577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674934431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10531030595302582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674948843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0979999303817749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674963110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313671827316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674977561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040899008512497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674991822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014974862337112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675006370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11230514943599701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675006370, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675006371, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675088059, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783035933971405, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675088060, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675088060, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784675102439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10802280157804489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675116927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10505212098360062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675131165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183759778738022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675145631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815508663654327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675159979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201860219240189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675174607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09684115648269653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675189174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213243216276169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675203527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086457222700119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675217874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099928081035614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675232315, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0944579616189003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675246751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371987521648407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675261505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181461811065674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675276077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118744522333145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675290356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778084188699722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675290356, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675290357, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675371461, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7825695276260376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675371462, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675371462, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784675386073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09979505091905594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675400539, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09672880172729492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675414894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611379891633987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675429663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126936823129654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675444342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11648958176374435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675458847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09589564800262451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675473389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058894544839859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675487786, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076646506786346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675502534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012304037809372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675516860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442431271076202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675530962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661626607179642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675545874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934396833181381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675560619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09836797416210175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675575516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283436417579651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675575516, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675575517, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675658152, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833617329597473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675658153, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675658154, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784675673084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149994283914566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675688085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027113348245621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675702892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10599249601364136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675717660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1182778999209404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675732393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115727499127388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675756242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038523375988007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675771336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750467121601105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675785930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11383799463510513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675800830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1059081181883812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675815644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678073018789291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675830498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10668173432350159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675845286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050955057144165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675860221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653004795312881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675874643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583202332258224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675874644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675874644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675957933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7833693623542786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675957935, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675957935, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784675972784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11201543360948563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675987373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10318779200315475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676002101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10204292088747025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676016909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599981039762497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676031838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704582929611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676046600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482965409755707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676061538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10326029360294342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676076326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129300594329834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676091563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1177036240696907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676106421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411347448825836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676121256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454513669013977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676136183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120765432715416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676150669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10767719894647598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676165525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11542575806379318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676165525, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676165526, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676246060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837756276130676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676246061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676246061, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784676260568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10094389319419861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676275254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035434901714325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676290156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11419522017240524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676304907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949523210525513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676319969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09871844947338104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676334850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700494050979614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676349604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855116158723831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676364304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880608856678009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676379141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996505618095398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676393524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563582926988602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676407961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481830477714539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676422454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003325134515762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676436850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750580370426178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676451574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199474334716797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676451574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676451575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676533901, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843025922775269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676533902, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676533902, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784676548580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11099579185247421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676563265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498686134815216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676577731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973504185676575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676592461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115827776491642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676607076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214172095060349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676621986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649005860090256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676636985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09867721050977707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676652030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078985407948494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676667277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743077099323273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676682346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074950322508812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676697611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11785858869552612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676712655, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107008308172226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676727625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132909566164017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676742808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034766584634781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676742808, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676742809, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676823949, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.783912718296051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676823950, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676823950, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784676838405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551365464925766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676853246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098114401102066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676867979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878921836614609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676883103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10597165673971176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676897830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528165847063065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676912858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118559002876282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676928405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591786354780197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676943390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10915226489305496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676958412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286942660808563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676973273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09277395159006119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784676988238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11631309986114502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677003214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137784481048584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677017750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09838394820690155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677032369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780985653400421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677032370, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677032370, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677114375, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7837282419204712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677114376, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677114376, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784677129174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102616250514984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677144174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09270666539669037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677159020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09539666771888733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677173944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843439400196075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677188905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734610259532928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677203849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126076728105545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677218598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09500134736299515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677233508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535935312509537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677248474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541889071464539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677263253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077417433261871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677278143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09797481447458267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677293483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826315194368362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677308562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088419035077095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677323389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09855497628450394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677323390, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677323390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677407209, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843371033668518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677407210, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677407211, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784677422520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1054578423500061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677437555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09549754112958908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677452669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10309574007987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677468005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09557663649320602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677483337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282792150974274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677498322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206608265638351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677513424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10511045902967453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677528298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09467200189828873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677543377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438558250665665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677558431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10366898775100708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677573020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279970616102219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677587989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10085694491863251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677602499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264488309621811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677617212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105595737695694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677617213, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677617213, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677699385, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836166620254517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677699386, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677699386, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784677714198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952596575021744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677728989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.099375881254673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677743704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007758229970932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677758132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017361730337143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677772875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09889489412307739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677787489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991870820522308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677801888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09653697907924652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677816306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09879887849092484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677830913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349613428115845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677845498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.107472263276577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677859993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305097699165344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677874544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10743246972560883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677889261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183820873498917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677903608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09505340456962585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677903609, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784677903609, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784677986606, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7836195230484009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784677986607, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784677986607, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678001290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09403159469366074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678015829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09876471757888794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678030458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09642671048641205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678044936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10000067204236984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678059357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370556265115738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678074351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09507767111063004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678088957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1018526628613472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678103666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09764290601015091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678117971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09755999594926834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678132426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349329560995102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678146740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09364781528711319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678161552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10727233439683914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678176471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10350541770458221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678191663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084885522723198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678191664, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678191664, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678273662, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843475341796875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678273663, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678273663, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784678288409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573210567235947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678303053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10119477659463882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678317663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464806109666824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678332646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232799500226974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678346856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09606719762086868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678361202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467879474163055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678376129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09481097757816315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678391047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389254778623581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678406104, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09792779386043549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678421649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09020402282476425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678436934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984034299850464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678452009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10486840456724167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678466891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043170616030693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678481832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926855355501175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678481833, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678481834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678563770, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843266129493713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678563771, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678563771, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784678578550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10015271604061127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678593188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508134961128235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678607836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09729056060314178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678622701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09956293553113937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678637675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10006140172481537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678652302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09359639137983322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678666998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402557253837585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678682161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212843865156174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678696739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09967504441738129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678711810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967189073562622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678726421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09763708710670471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678741548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09983526170253754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678756512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966250509023666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678771253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509949922561646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678771254, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678771254, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678852115, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7843419909477234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678852116, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678852116, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784678866962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10071229934692383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678881835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09461238980293274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678896702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09786321222782135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678911337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729039460420609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678926427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10293794423341751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678941248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08700121194124222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678956382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10442081838846207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678971449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09255798906087875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678986587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467340797185898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679001822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09510897845029831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679016934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09473434090614319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679031900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10254229605197906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679047076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097356677055359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679061957, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09735143929719925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679061957, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679061958, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679146572, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7850779891014099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679146573, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679146573, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784679160943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09530860930681229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679175404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09564737230539322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679189743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068672895431519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679204108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10249212384223938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679218811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331883281469345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679233507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497929155826569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679248460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10864897072315216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679262833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11488619446754456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679277406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778601467609406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679292017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09840866178274155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679306585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058306694030762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679321121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0961480662226677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679335644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09434182196855545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679350185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896923393011093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679350185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679350186, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679434550, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7847976684570312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679434551, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679434551, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1784679449222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09868940711021423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222658560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679464029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417931318283081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222822400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679478632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562597215175629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222986240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679493307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09244155883789062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223150080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679508106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344358533620834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223313920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679523004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0973079577088356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223477760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679537850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09469249099493027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223641600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679552719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10518466681241989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223805440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679567343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036195456981659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 223969280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679581904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09530769288539886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224133120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679596848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602153837680817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224296960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679611633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09457076340913773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224460800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679626516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09760872274637222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224624640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679641000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09849347174167633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224788480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679641000, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679641001, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679723006, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7852804660797119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679723007, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679723007, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 224788480, "epoch_num": 0.09812511092814545}} +:::MLLOG {"namespace": "", "time_ms": 1784679737975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739230364561081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 224952320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679752702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219293087720871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225116160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679767479, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475054383277893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225280000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679782641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053666681051254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225443840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679797525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09687643498182297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225607680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679812026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10623563826084137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225771520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679827020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10629212856292725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 225935360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679841590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676763206720352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226099200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679856459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835833847522736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226263040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679871432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426422953605652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226426880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679885922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11001455783843994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226590720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679900405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11179947108030319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226754560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679915692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09257249534130096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 226918400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679930639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408957302570343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227082240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679930640, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784679930641, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784680011847, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7855668663978577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784680011848, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784680011848, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 227082240, "epoch_num": 0.09912638757026938}} +:::MLLOG {"namespace": "", "time_ms": 1784680026923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1009024977684021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227246080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680042436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10431569069623947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227409920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680057697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11286664754152298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227573760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680073260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703492164611816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227737600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680088778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10584703087806702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 227901440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680104050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875824093818665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228065280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680119460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11104506999254227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228229120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680134911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150852799415588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228392960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680150276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973231494426727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228556800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680165814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699834674596786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228720640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680180986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129950195550919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 228884480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680196153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11156036704778671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229048320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680211309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10697250813245773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229212160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680226530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550966113805771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 229376000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680226531, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784680226531, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 229376000, "epoch_num": 0.10012766421239332}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed770796222.log b/recommendation_v4/rcp_logs/gbs_8192/seed770796222.log new file mode 100644 index 000000000..f1fc24ecc --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed770796222.log @@ -0,0 +1,684 @@ +:::MLLOG {"namespace": "", "time_ms": 1784939247303, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784939247303, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784939268454, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784939268455, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784939268455, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "seed", "value": 770796222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784939268456, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784939268457, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784940110137, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784940110137, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784940110497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784940225888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13908115029335022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784940238689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829296827316284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940251849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384357213973999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940265251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382930427789688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940278733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386009007692337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940292201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823385536670685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940305614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383131444454193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940319334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871265947818756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940333013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860803842544556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940346429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859035074710846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940360312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851207494735718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940374023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13880814611911774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940387889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882553577423096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940401489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13926050066947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940401489, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940401489, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940483201, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998165965080261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940483202, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940483202, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940496432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840055465698242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940509927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842704892158508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940524066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828259706497192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940537480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825000822544098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940551164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795754313468933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940565145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13807256519794464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940578514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793312013149261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940592089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137899249792099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940605997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13773825764656067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940619442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816820085048676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940633402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382180154323578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940647369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384699046611786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940661094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784074783325195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940675276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382867693901062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940675276, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940675277, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940758347, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.500006914138794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940758349, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940758349, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940771860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771642744541168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940785388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823577761650085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940798897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13701128959655762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940813016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366855502128601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940826669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379123330116272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940840257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13788297772407532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940853946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13718225061893463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940867818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368478238582611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940881412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365009993314743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940894857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766390085220337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940908535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13720516860485077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940921873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761404156684875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940936274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777071237564087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940950145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373598873615265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940950146, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784940950146, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941032018, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5002294778823853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941032018, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941032019, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941045834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363908052444458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941059877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13721664249897003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941073729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645799458026886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941088209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13699404895305634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941101923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730719685554504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941115920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13769389688968658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941129816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13629816472530365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941143620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372116059064865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941157914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656985759735107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941171983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863030076026917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941186138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614261150360107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941200117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13786545395851135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941214128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506758213043213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941228298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13670232892036438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941228299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941228300, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941309336, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5005543828010559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941309337, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941309337, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941322887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361103653907776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941336494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348491907119751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941349887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771997392177582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941363597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402718305587769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941377447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645735383033752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941391026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282491266727448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941404707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524264097213745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941418391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512462377548218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941432133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793335855007172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941445769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347508728504181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941459546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1485426425933838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941473593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532310724258423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941487229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13346447050571442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941500706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783738017082214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941500707, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941500707, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941582026, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5009433031082153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941582026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941582027, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941595510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505534827709198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941609226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14002081751823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941623116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576224446296692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941637086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282644748687744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941651148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1346132606267929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941665316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13948161900043488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941679219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303864002227783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941693210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259045779705048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941707099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295911252498627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941720661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309451460838318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941734525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13067713379859924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941748089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13475008308887482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941761995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131047785282135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941776259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13924817740917206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941776260, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941776260, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941857384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5014552474021912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941857385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941857385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941871401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351845741271973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941885349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332426518201828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941899456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13579390943050385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941913126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349928081035614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941926882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381620168685913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941940766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12743262946605682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941954263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524407148361206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941968086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12914790213108063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941982013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317517936229706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941996085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220351934432983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942009924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336405158042908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942024032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428744673728943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942038920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353127360343933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942053397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355353474617004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942053397, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942053398, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942134869, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5022715330123901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942134869, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942134870, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942149299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1325080543756485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942163739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337248980998993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942178401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13541705906391144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942193174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538599014282227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942207682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13978520035743713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942221879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331697404384613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942236599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533172011375427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942250622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345962762832642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942264602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326501846313477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942278844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162453472614288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942292951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13565203547477722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942307515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13349440693855286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942321721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321444392204285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942335734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050124049186707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942335735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942335735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942417803, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033178329467773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942417804, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942417805, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942432708, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13784071803092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942447078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348913460969925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942461767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533581793308258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942476524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310604214668274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942490763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335025131702423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942505345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13425405323505402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942520143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13724718987941742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942534791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397210836410522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942549373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295944035053253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942563642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944483757019043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942577716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275712549686432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942591881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13303858041763306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942606191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945109605789185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942620084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12461645156145096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942620085, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942620085, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942700644, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5043933987617493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942700645, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942700645, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942714942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101999461650848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942729149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360672414302826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942743571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1333436369895935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942757950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13029563426971436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942771857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12814484536647797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942786038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12663990259170532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942800695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12885528802871704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942815050, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13229918479919434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942829230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521791994571686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942843412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12488709390163422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942857435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12857326865196228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942871497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344846487045288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942885665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097885251045227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942900199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12945348024368286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942900199, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942900200, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942983060, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058243274688721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942983061, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942983061, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784942997547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13077408075332642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943011599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12903596460819244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943025275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308469951152802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943039622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12399835139513016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943054000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12955829501152039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943067881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12581558525562286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943081865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13009564578533173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943096108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198678404092789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943109981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594888150691986, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943124243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716500997543335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943138363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12368225306272507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943152968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321016192436218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943167285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568898499011993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943181493, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12916119396686554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943181493, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943181494, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943264499, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5074609518051147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943264500, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943264500, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943278425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364949494600296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943293110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647565245628357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943306946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358993500471115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943320768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14011676609516144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943334810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674508035182953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943348884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819822669029236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943362862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12736250460147858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943376881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12607093155384064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943390884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14213509857654572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943405144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14093175530433655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943419395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12720386683940887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943433541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12792842090129852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943447610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237667083740234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943461219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315924972295761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943461220, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943461220, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943545784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5092635750770569, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943545785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943545785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943560660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1412469744682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943575245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14236417412757874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943589635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353716105222702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943604159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955093920230865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943618791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14500051736831665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943632969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409694254398346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943647451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1261337846517563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943661912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771438598632812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943676612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317090094089508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943691287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627910614013672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943705855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770975172519684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943720082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133291557431221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943735007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1344006359577179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943749707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364137828350067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943749708, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943749708, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943832268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5111699104309082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943832269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943832269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943846438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857853412628174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943861025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825443387031555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943875630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175275921821594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943890255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292165100574493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943904739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378921151161194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943919419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13955466449260712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943933949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326322704553604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943948624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14118103682994843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943962686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14083534479141235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943977467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512617349624634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943991903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.127433180809021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944006186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300075799226761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944020459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13402236998081207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944034845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12760134041309357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944034846, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944034846, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944116588, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5134813785552979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944116589, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944116589, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944130747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13232794404029846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944144926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13679839670658112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944159240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366927206516266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944173646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192607462406158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944188247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337735950946808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944202766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347522914409637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944217590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13388365507125854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944232151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312214612960815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944246456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13986335694789886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944261249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190999627113342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944275657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12872928380966187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944289740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112932443618774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944303937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13251857459545135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944318304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551901280879974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944318305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944318305, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944400279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5165881514549255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944400280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944400280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944414766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326072692871094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944429177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12302582710981369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944443619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051612675189972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944458113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13037177920341492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944472537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512372970581055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944487586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540875911712646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944502131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539008796215057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944516988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418771535158157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944531546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093052804470062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944545836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119067251682281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944560699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302916556596756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944575003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333092629909515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944589373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080580532550812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944603363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1300952136516571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944603363, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944603364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944685211, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5207803249359131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944685213, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944685213, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944699586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377429962158203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944713904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236793875694275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944728199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350196182727814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944742516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306360363960266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944756636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270102858543396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944770694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13224314153194427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944785169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574163615703583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944799740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13292191922664642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944814431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312902271747589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944829175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13902471959590912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944843712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377511501312256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944857760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375884711742401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944872727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193300366401672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944887360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331569731235504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944887361, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944887361, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944970128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5264279842376709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944970129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944970129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784944984744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13132791221141815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944999230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487935066223145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945013764, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770917057991028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945028256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341452181339264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945042647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726476788520813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945057148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291874647140503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945071556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12599241733551025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945085933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12383176386356354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945100593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615980744361877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945115322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374301165342331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945129974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13457061350345612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945145123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12706312537193298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945159843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13096767663955688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945174030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12366443872451782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945174031, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945174031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945258685, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5341439843177795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945258686, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945258686, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945273434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327822059392929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945287694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13257776200771332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945302017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339135766029358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945316282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797128200531006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945330573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360999345779419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945344852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13139860332012177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945358988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272415816783905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945373196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13500526547431946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945387261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12464774399995804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945401761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13031405210494995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945416168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240061700344086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945430695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13430683314800262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945445298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1404537558555603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945459732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362384408712387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945459732, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945459733, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945542880, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5457786917686462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945542880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945542881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945557941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642795383930206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945572715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13140136003494263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945586956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13258618116378784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945601573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1393444687128067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945617162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298486292362213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945632579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383754312992096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945647501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13501916825771332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945662559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712294399738312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945677340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494929671287537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945692095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1307007521390915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945706831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14053502678871155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945721433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13462436199188232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945735926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921643257141113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945750556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331720232963562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945750557, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945750557, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945833833, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5595343708992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945833834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945833834, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945848607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1421358585357666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945863279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327095776796341, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945878422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331631988286972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945893093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091619312763214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945908132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524457812309265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945923046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378183543682098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945937807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396644592285156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945952704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559584319591522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945967374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312534064054489, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945982284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290619850158691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945996886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314951956272125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946011628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717882335186005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946026008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13646341860294342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946040811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375761181116104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946040811, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946040812, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946123536, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5758647322654724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946123538, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946123538, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946138224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154739141464233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946153130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355418711900711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946167885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13410288095474243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946183048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237182796001434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946197675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371123194694519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946212355, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296699494123459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946227289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369764387607574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946241392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12375826388597488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946255621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104352355003357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946269739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182342052459717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946283927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312585175037384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946298124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13183942437171936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946312428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290494203567505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946326839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097837567329407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946326839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946326840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946410958, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5904187560081482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946410959, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946410959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946424899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224183440208435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946439250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12711462378501892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946453378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12965339422225952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946467621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335100382566452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946482298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13196958601474762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946496572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222563311457634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946511165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13090331852436066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946525564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13220496475696564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946539832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373385637998581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946553780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830688059329987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946568717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356652855873108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946583195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11812744289636612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946597530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12209746986627579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946611961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257966160774231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946611961, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946611963, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946694449, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.605269730091095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946694450, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946694450, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946708610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196754664182663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946722652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12777233123779297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946736771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12911397218704224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946750945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322184056043625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946765025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12190928310155869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946779293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521234691143036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946793449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995512783527374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946807523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12219559401273727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946821873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276315450668335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946836536, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750087678432465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946850985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11887017637491226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946865850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317559778690338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946880410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276593953371048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946894901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289098858833313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946894901, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946894901, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946978259, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6186714768409729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946978260, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946978260, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784946992274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12892256677150726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947007256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11932592839002609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947021590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12278418987989426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947036334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12184914201498032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947050796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239960715174675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947065631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12649443745613098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947080159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172710582613945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947094686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12313637137413025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947109294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12562932074069977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947123993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12458573281764984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947138296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.116246797144413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947152847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12487556040287018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947167254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108582466840744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947181637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12549887597560883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947181638, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947181638, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947265604, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6296008825302124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947265605, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947265605, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947280180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11771519482135773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947294789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1153372973203659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947309806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034747540950775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947324814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12259611487388611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947339421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12458934634923935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947353873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172061562538147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947367892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12063441425561905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947382614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12274697422981262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947396900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12143584340810776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947411248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318125575780869, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947425563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12137096375226974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947439641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12246861308813095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947453809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12525615096092224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947468228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11755955219268799, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947468229, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947468229, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947554067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6432031989097595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947554067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947554068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947568244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12023003399372101, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947582227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11267684400081635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947596208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11055552959442139, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947610270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370252072811127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947624308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10931577533483505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947638473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12218620628118515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947652921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696108430624008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947667706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210312694311142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947682126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356951504945755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947697001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11947640031576157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947711675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11294088512659073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947726265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11368671804666519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947740666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027511954307556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947755357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599870771169662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947755358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947755358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947842120, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6611868739128113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947842121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947842121, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784947856891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12149536609649658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947871385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11984942108392715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947886164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12389393150806427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947900840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601567476987839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947915556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471689492464066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947930027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780353635549545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947944706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11966494470834732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947959228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11981038749217987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947973590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10898194462060928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947987953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11845295876264572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948002367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586112529039383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948016333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11705241352319717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948030538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11378757655620575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948045264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12543413043022156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948045264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948045265, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948130879, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6810621619224548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948130880, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948130881, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948145466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.124947190284729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948159923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11705400049686432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948174062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11691385507583618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948188695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222459003329277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948203004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110455222427845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948217377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11143757402896881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948231350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11309821158647537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948245954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866034775972366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948260371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174283921718597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948275498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11624515056610107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948290180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12413394451141357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948304799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545594781637192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948319659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12752750515937805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948334425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136818528175354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948334426, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948334427, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948421598, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7033082246780396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948421599, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948421599, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948436466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726053595542908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948451087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11462090909481049, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948465809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1160336434841156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948480720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11386723816394806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948495167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1204114556312561, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948509917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11917835474014282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948524983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12184139341115952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948539910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12076765298843384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948554673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155557855963707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948569907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352156102657318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948584713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11603126674890518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948599415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11703617870807648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948614099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12658073008060455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948629217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11687831580638885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948629217, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948629218, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948715612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7209222912788391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948715613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948715613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948730405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12301886081695557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948745492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1208970695734024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948760305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11379323154687881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948775263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11600662022829056, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948789959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011282354593277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948804731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673440784215927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948819626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925292760133743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948834144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11284158378839493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948849089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10967691987752914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948863895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10930924117565155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948879091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235555037856102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948894420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707709938287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948909471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12104865908622742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948924781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12014001607894897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948924782, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784948924783, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949010384, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7403948903083801, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949010385, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949010385, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949025450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11940175294876099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949040347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1191188171505928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949055210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11645843833684921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949070225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11752833425998688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949084790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039857566356659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949099419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515772551298141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949113999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106933444738388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949128476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11533806473016739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949142982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491164028644562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949157496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330644994974136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949171826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755132883787155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949186354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132061555981636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949200925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11391036957502365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949215342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11702322959899902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949215381, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949215381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949298244, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7518444061279297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949298245, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949298245, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949312665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10527181625366211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949327515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760879516601562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949342497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11651502549648285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949357198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11225991696119308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949372060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859246551990509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949386933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11640329658985138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949401602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11371051520109177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949416180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442621052265167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949430903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720287799835205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949445367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219248175621033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949459783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.100554920732975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949474305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457717418670654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949488659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897984355688095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949503188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10640757530927658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949503189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949503189, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949586816, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7571488618850708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949586818, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949586818, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949601102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836154967546463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949615773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11430785804986954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949630298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742151945829391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949644852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10466013848781586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949659547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11370434612035751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949674089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10755450278520584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949688977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08868443965911865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949703696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929553955793381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949718428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027444377541542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949733264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11751611530780792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949748313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806339979171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949763085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070987790822983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949778423, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126803070306778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949793558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778351128101349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949793559, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949793559, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949877089, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7598263025283813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949877089, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949877090, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784949891999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708478629589081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949906812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126629188656807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949921025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10418039560317993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949936074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077382043004036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949950509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984703361988068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949965105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448721796274185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949979572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723063349723816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949994287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10949819535017014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950008736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09775824099779129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950023426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10901482403278351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950037940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10503822565078735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950052481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086205244064331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950067116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893835872411728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950081903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113210529088974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950081904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950081904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950166267, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.760788083076477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950166268, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784950166268, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed79275716.log b/recommendation_v4/rcp_logs/gbs_8192/seed79275716.log new file mode 100644 index 000000000..e588fcb08 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed79275716.log @@ -0,0 +1,662 @@ +:::MLLOG {"namespace": "", "time_ms": 1784939249673, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784939249673, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784939271351, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784939271358, "event_type": "POINT_IN_TIME", "key": "seed", "value": 79275716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784939271359, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784939271359, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784939271359, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784939271359, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784939271498, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784939271498, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784940213661, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784940213661, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784940214093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784940333193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384589970111847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784940345653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838879764080048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940358548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802246749401093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940371760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874493539333344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940384891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138676717877388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940398255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13875487446784973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940411452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855965435504913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940425725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382516622543335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940439088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1386032998561859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940452247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859611749649048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940466063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872095942497253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940479680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867543637752533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784940493368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385490745306015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940506853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138570174574852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940506853, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940506854, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940591537, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5028355121612549, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940591539, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940591539, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784940605183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858914375305176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940618766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861636817455292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940632555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13829097151756287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940645816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851116597652435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940659428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13831153512001038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940673574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13846313953399658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940687039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834866881370544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940700839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138217955827713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940715140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823214173316956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940728763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13828836381435394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940742903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845321536064148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940757110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863115012645721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940771007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818883895874023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940785417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853207230567932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940785418, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940785418, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940872642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030320882797241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940872643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940872643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784940886525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13810506463050842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940900325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13845384120941162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940915002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374104619026184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940929451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723191618919373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940943298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805048167705536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940957249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794691860675812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940971145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376354694366455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940985409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730959594249725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784940999169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13682641088962555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941012839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733188807964325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941026790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13734084367752075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941040421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13759243488311768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941055133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772623240947723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941069223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375669538974762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941069224, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941069224, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941154052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032646656036377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941154053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941154053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784941167806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13681137561798096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941181920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772065937519073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941195946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13677994906902313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941210421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13756728172302246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941224194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373857706785202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941238156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764873147010803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941252121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366317719221115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941265933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13754764199256897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941280421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690176606178284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941294905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860411942005157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941309230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363380253314972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941323368, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802608847618103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941337564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531439006328583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941351943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651034235954285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941351944, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941351945, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941436612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5036002397537231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941436613, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941436613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784941450297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606128096580505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941464007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13549883663654327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941477639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372385323047638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941491554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444137573242188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941505530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658683001995087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941519597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13344912230968475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941533631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545337319374084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941547471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13545702397823334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941561563, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13766984641551971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941575281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505448400974274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941589181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14724452793598175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941603420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540728390216827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941617388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1337776631116867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941631293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772110641002655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941631294, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941631294, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941715490, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5039758682250977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941715492, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941715492, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784941729290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135182186961174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941743081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13948921859264374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941757040, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359054446220398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941770960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332324743270874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941784894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13487255573272705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941799071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13909024000167847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941815770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333703577518463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941829820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13276197016239166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941843930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308610022068024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941857711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13144132494926453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941871719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094298541545868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941885442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484132289886475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941899298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13164958357810974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941913612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898012042045593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784941913612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941913613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941997527, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5044925212860107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941997528, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784941997528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784942011538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361403346061707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942027956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13313068449497223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942042276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13575048744678497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942056343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490363955497742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942070431, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374143660068512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942084639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276223361492157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942098506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536088168621063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942112658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12913143634796143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942126918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326281309127808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942141452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253575563430786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942155642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355928659439087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942170239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13428160548210144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942185034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530606031417847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942199289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332223892211914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942199289, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942199289, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942287470, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5053205490112305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942287471, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942287471, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784942301942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13238157331943512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942316218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13367578387260437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942330721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555413484573364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942345496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135372593998909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942360085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396757960319519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942375686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332206279039383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942391171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534988462924957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942405900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342468440532684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942420373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305146992206573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942435124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318422257900238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942449589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13564135134220123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942464665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1336050182580948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942479356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352692127227783, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942493752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046236336231232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942493753, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942493753, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942580083, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5064427852630615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942580084, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942580084, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784942595159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380176842212677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942609582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492722809314728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942624121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13533683121204376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942638657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127614557743073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942653045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355574011802673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942667530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399839401245117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942682252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710105419158936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942696760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13387475907802582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942711372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266240060329437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942725898, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12963411211967468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942740058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12744566798210144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942754362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330169141292572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942768738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12947812676429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942782829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12467687577009201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942782829, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942782830, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942870920, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5076387524604797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942870921, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942870921, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784942885397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13100849092006683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942899704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13599468767642975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942914319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317906856536865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942928949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033026456832886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942943174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282762587070465, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942957718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12640240788459778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942972481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289229393005371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784942986680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13212737441062927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943001007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352493166923523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943015325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248716488480568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943029480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12832552194595337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943043620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1343757063150406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943058136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13081835210323334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943072903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12922723591327667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943072904, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943072904, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943160641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5091302990913391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943160642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943160642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784943175357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13051624596118927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943189851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12906013429164886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943203871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309800624847412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943218600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12377951294183731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943233192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1295517385005951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943247433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12575110793113708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943261603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298164427280426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943275944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12179435789585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943290055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596394658088684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943304649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13724344968795776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943318981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354634702205658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943333750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12318979948759079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943348398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12551932036876678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943362690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1291576474905014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943362691, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943362691, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943449734, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5108367204666138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943449735, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943449736, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784943465736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364586353302002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943480412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136612206697464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943494376, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576073944568634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943508320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13996271789073944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943522513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665401935577393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943536818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382865607738495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943550926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273144632577896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943565154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606240808963776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943579307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14227518439292908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943593938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14113564789295197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943608529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12727729976177216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943622948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12833034992218018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943637293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250784575939178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943651003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13150382041931152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943651004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943651004, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943735948, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5128982663154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943735949, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943735949, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784943751221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14137209951877594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943766212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14223109185695648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943780787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502879440784454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943795691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1395556777715683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943810722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14479726552963257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943825466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14097802340984344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943840575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12613138556480408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943855595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375386267900467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943870718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318052113056183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943885882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13602755963802338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943900876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793987035751343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943915602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287483155727386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943930712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13435012102127075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943945482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635076582431793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784943945483, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784943945483, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944031929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5150726437568665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944031930, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944031930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784944046336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13835997879505157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944061079, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797283172607422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944075828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1318012773990631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944090516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13286541402339935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944105060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381525874137878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944120002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396256536245346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944134881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13249988853931427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944149894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14113011956214905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944164216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14074121415615082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944182744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13505703210830688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944197581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12746065855026245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944212026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.129940003156662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944226719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13406094908714294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944241643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12767982482910156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944241644, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944241644, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944324695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5177190899848938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944324696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944324696, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784944339443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321127712726593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944354078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691186904907227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944368907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357096910476685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944383685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13191959261894226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944398618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13391011953353882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944413300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13334967195987701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944428500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373778760433197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944443359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275274634361267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944457766, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.139910027384758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944472829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178357481956482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944487481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12881991267204285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944502019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13080760836601257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944516531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210564851760864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944531175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13587018847465515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944531176, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944531176, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944616697, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5211915969848633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944616698, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944616698, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784944631385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13332587480545044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944645905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12292779982089996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944660404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043265044689178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944675073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303083300590515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944689569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531696796417236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944704636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13530273735523224, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944719303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527308404445648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944734432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12395865470170975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944749172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309892237186432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944763407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312272995710373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944778347, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302596479654312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944792862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308760523796082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944807247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13091112673282623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944821353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1299605667591095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944821353, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944821354, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944907793, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5259149670600891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944907794, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944907794, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784944922182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777679204940796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944936651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13245578110218048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944951042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327381014823914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944965372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13034884631633759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944979791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269235759973526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944994107, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202351331710815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945008945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354610174894333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945023770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1327337622642517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945038628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117243349552155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945053624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13894081115722656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945068535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133607417345047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945082749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372377872467041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945099541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1319122612476349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945114030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331419050693512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945114031, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945114031, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945196805, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5322238206863403, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945196805, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945196806, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784945211255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13105788826942444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945225488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1349133402109146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945239654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779981434345245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945253933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318198919296265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945268066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1273028552532196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945282243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292535811662674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945296375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12582583725452423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945310422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12386692315340042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945324932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596948981285095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945339344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13711674511432648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945353804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13456317782402039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945368851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1268204152584076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945383448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13084052503108978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945397543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12340342253446579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945397544, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945397544, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945483268, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.540954053401947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945483269, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945483269, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784945498009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13226433098316193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945512462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326056867837906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945526925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335318386554718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945541465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13761144876480103, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945555909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359250545501709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945570550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312418282032013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945585068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699148058891296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945599601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13490253686904907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945613848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12461409717798233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945628585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301630586385727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945643176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237299025058746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945657819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13398495316505432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945672409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14014899730682373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945686934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623595237731934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945686934, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945686935, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945773331, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5536198019981384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945773332, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945773332, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784945788424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13654080033302307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945803140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133756816387177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945817411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13215646147727966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945832047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13890017569065094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945847116, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12981411814689636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945862237, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13821111619472504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945876942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348700225353241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945891897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13717760145664215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945906685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13466227054595947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945921619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045468926429749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945936534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14027011394500732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945951272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13464538753032684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945966164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12887316942214966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945981019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13312436640262604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945981020, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784945981020, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784946066007, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5682582259178162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784946066008, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784946066008, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784946081192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14176234602928162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946096037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326344609260559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946111253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13341595232486725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946126110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064800202846527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946141780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13522440195083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946156902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327041268348694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946171624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374803960323334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946186386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354609578847885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946200985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13127972185611725, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946215880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13240739703178406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946230751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280631601810455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946245641, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13703078031539917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946260241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13569779694080353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946275184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13731378316879272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946275184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946275185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946359929, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5837602615356445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946359929, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946359930, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784946374608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314375400543213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946389688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13508403301239014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946404586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13416247069835663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946419964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13169914484024048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946434805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702024519443512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946449579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12917961180210114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946465084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1331995725631714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946479517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1233440712094307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946494137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046760857105255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946508777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311265230178833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946523513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097478449344635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946537974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310390681028366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946552447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880928814411163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946566971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055020570755005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946566972, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946566973, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946649275, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5961446762084961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946649276, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946649276, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784946663597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1219489574432373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946678069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12650297582149506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946692522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128843754529953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946706995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279326260089874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946721846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121843338012695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946736457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1214793249964714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946751521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301001012325287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946766173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13194319605827332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946780826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13637301325798035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946795018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12805326282978058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946810258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324310600757599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946825065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174347847700119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946839734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12090972065925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946854598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12466387450695038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946854599, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946854599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946940740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6099399328231812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946940741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946940741, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784946955294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12136825919151306, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946969697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272234171628952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946984341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12792085111141205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946998935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125501573085785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947013472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12123528867959976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947028065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338879019021988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947042520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12944987416267395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947056934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12141864746809006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947072006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693150341510773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947086919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12692146003246307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947101623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813520640134811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947116776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1313779503107071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947131643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.126569926738739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947146733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812970578670502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947146734, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947146734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947230473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6242638826370239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947230474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947230474, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784947244648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824563682079315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947259818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872322857379913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947274445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1219017431139946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947289086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12122499942779541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947303437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12375984340906143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947317897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12556768953800201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947332415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159924641251564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947346964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12250314652919769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947361746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12498848885297775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947376289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12439394742250443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947390572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11538880318403244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947405774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12472108751535416, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947420340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965210199356079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947435072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12401672452688217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947435072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947435073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947518671, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6381000280380249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947518672, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947518672, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784947533722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1167401671409607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947548647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433231830596924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947563446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830066680908203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947578594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12151102721691132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947593366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12402579933404922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947608100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617665737867355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947622399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12001252174377441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947637552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187697738409042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947652244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210252195596695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947666895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272695660591125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947681625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1215846836566925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947696145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12171158939599991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947710524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12547388672828674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947725221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709696799516678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947725221, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947725221, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947808217, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.659315824508667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947808218, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947808218, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784947822826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021778523921967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947837610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118880957365036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947852172, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042608320713043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947866625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136852577328682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947881397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10926269739866257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947895978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12188941240310669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947910808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590326577425003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947925989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039399892091751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947940622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360760241746902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947955716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11948325484991074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947970709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11349905282258987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947985534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281759291887283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948000122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897571593523026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948014747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11634759604930878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948014748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948014748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948100592, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6920140385627747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948100593, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948100593, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784948115358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12061599642038345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948130780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872785538434982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948146018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12275169789791107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948160946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10612951964139938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948175909, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394484341144562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948190724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1181885227560997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948205950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11869572848081589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948221027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922179907560349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948235875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853414982557297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948250849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821328848600388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948265884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409124732017517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948280548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11646851897239685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948295295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11278320848941803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948310348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12488026916980743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948310348, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948310349, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948397052, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7236897349357605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948397053, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948397053, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784948412077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312706559896469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948427100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11757326126098633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948441918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546124517917633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948457054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12162691354751587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948471912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825000703334808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948486799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062552779912949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948501181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11277886480093002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948516232, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863063484430313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948531060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075796186923981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948546425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11657357215881348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948561494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12307686358690262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948576413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11446765065193176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948591654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12574386596679688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948606818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263883858919144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948606818, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948606819, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948694179, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.757096529006958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948694181, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948694181, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784948709233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12484127283096313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948723910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313269287347794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948738783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485761404037476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948753921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11211565136909485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948768927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784689873456955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948783977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155950054526329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948799409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12123489379882812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948814729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11998263746500015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948829875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11394039541482925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948845598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202368140220642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948860738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479896306991577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948875837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239942908287048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948890687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12354546785354614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948906151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11490156501531601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948906151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948906152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948997207, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7623125314712524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948997208, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784948997208, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784949012319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022266536951065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949027631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11714844405651093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949042638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1125977411866188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949058010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11314524710178375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949073083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751591622829437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949088031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11341077834367752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949103167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11698585748672485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949117993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10953201353549957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949133068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871341824531555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949148043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732739418745041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949163360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12153635919094086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949178802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10457732528448105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949194018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951538920402527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11947057396173477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209546, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949209546, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949297623, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7651932835578918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949297624, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949297624, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784949312913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11719481647014618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949327933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590303480625153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949342974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11536448448896408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949358261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.112989142537117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949372949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192976891994476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949387649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093126446008682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949402473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11744915693998337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949417166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11478514224290848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949432066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296855121850967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949446785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10078190267086029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949461229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594749450683594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949476140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151231825351715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949490893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10971289873123169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949505752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11367005854845047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949505800, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949505800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949592058, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7673619985580444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949592059, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949592059, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784949606781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404139012098312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949621946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105603888630867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949637382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134587973356247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949652246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962709039449692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949667547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10671421885490417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949682944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134776696562767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949697926, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132398396730423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949712955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081937700510025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949727983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481109261512756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949742575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10988499224185944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949757169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983113944530487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949771851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265081912279129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949786390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1061093658208847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949801068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10491935163736343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949801069, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949801069, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949886222, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7688021659851074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949886223, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949886223, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784949900844, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10519900918006897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949915528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124427393078804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949930310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09566386044025421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949944924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355310887098312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949959469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111878901720047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949973960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456492751836777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949988556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08834969252347946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950002970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854516178369522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950017612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09996579587459564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950032568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11731575429439545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950047636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072343960404396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950062462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963557124137878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950077830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09840378910303116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950093004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658617317676544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950093005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784950093006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed869764802.log b/recommendation_v4/rcp_logs/gbs_8192/seed869764802.log new file mode 100644 index 000000000..1cbd4c04c --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed869764802.log @@ -0,0 +1,1859 @@ +:::MLLOG {"namespace": "", "time_ms": 1785027754209, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1785027754209, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1785027774469, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "seed", "value": 869764802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1785027774470, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1785027774471, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1785027774471, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1785027774471, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1785027774471, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1785027774472, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1785028671186, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1785028671186, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1785028671541, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1785028788360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13898107409477234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1785028800588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13851670920848846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028812930, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853199779987335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028825398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387976109981537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028838073, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885189592838287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028850540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874991238117218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028863287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886554539203644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028876076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138556569814682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028888952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882212340831757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028901782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13869385421276093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028914732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864915072917938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028927822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859106600284576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1785028940892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385444849729538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785028953760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1394260972738266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785028953761, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785028953761, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029038092, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49874141812324524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029038093, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029038093, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1785029050560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138471320271492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029063275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13882774114608765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029076588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852284848690033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029089500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855212926864624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029102695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385209858417511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029116292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861756026744843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029129125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855499029159546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029142227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833901286125183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029155882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383262425661087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029169224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384800225496292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029182836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383853405714035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029196292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13860969245433807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029209580, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13826005160808563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029223471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876043260097504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029223471, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029223472, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029306262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4989405572414398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029306263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029306263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1785029319795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381968855857849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029333430, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823062181472778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029347135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13768407702445984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029361012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715051114559174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029374298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794060051441193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029387690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13816022872924805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029401122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777537643909454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029415266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373656541109085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029428912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370459645986557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029442294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372988522052765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029455920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1373705118894577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029469286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13794144988059998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029483741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378132700920105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029497282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378411501646042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029497282, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029497283, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029580136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4991629719734192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029580137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029580137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1785029593462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13712626695632935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029607064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13737013936042786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029620569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1368318498134613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029634667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13735729455947876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029648186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374334692955017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029661888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13752532005310059, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029675570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366948038339615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029689096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137497216463089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029703208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13693587481975555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029717043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861161470413208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029730755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13650110363960266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029744282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13805820047855377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029757968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13536369800567627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029772072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367400884628296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029772072, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029772073, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029855801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49948549270629883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029855802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029855802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1785029869226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1362951695919037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029882744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538837432861328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029896066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13716968894004822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029909724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13451749086380005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029923521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13659778237342834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029937129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13366007804870605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029950848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562755286693573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029964370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354185938835144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029978105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13792723417282104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785029991585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135382741689682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030005175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14739231765270233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030019062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548900187015533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030032460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13397835195064545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030046169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1379273235797882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030046170, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030046170, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030130391, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4998573958873749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030130392, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030130392, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1785030143869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13527265191078186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030157344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13968053460121155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030171092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13600970804691315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030184848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294439017772675, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030198709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494229316711426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030212877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13926194608211517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030226865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133244588971138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030241033, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330481469631195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030254642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317477703094482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030268115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13142292201519012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030281895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113345205783844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030295263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506627082824707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030308862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13156355917453766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030322765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879364728927612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030322765, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030322766, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030406187, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5003622770309448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030406188, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030406188, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1785030419986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338634043931961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030433626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325437903404236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030447757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357172280550003, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030461562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13471026718616486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030475375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137643963098526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030489294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275871992111206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030502912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354709416627884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030516791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292845904827118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030530781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356344401836395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030545062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323835849761963, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030559047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364167511463165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030573168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13440929353237152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030587170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537879288196564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030600992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326020538806915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030600993, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030600994, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030686128, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5011882185935974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030686129, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030686129, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785030699994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261064887046814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030713682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13370363414287567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030727478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13554790616035461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030741651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13523520529270172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030755630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1396695226430893, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030769354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13314925134181976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030784035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557963073253632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030798124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342292606830597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030812156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327716290950775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030826508, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1317211538553238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030840566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13585814833641052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030855102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378089666366577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030869289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13331890106201172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030883249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061809539794922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030883250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030883250, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030967365, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.502359926700592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030967366, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030967366, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785030982072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380193680524826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785030995971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347745805978775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031009895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13524289429187775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031024091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13112135231494904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031037983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13362663984298706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031052003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340661197900772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031066240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371760219335556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031080465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13395676016807556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1785031094850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13288359344005585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031108724, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12952518463134766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031122251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750637531280518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031136004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13282090425491333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031149938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12961708009243011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031163518, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12473078817129135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031163518, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031163519, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031246680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035377144813538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031246681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031246681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1785031260690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110056519508362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031274516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13590505719184875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031288623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328440487384796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031302675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13026708364486694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031316323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12829077243804932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031330233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12661612033843994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031344210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12900416553020477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031357738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321902722120285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031371303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352188140153885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031385013, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12477665394544601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031398442, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12856657803058624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031411910, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433955609798431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031425440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13098841905593872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031439313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12907272577285767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031439314, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031439314, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031521680, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.505046546459198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031521681, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031521681, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1785031535631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13043121993541718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031549298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1290569007396698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031562619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13336819410324097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031576961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12373275309801102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031591164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12954549491405487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031604852, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1257096379995346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031618687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989862263202667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031632647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12171173095703125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031646377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13574789464473724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031660560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13728168606758118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031674410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12346401065587997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031688627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299896031618118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031702711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254303902387619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031716662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12891677021980286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031716662, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031716662, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031799632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.506938099861145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031799633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031799633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1785031813372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363847255706787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031827399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1365797072649002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031840812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357940435409546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031854150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1398850977420807, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031867618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13667358458042145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031881149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13833211362361908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031894497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723608314990997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031907845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12606807053089142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031921186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14222481846809387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031935026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1410762518644333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031948882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726807594299316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031962587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12848599255084991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031976135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279594480991364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031989250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13146823644638062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785031989250, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785031989251, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032071995, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5090455412864685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032071996, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032071996, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1785032086664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14160659909248352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032100868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14239580929279327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032114751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352817565202713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032128855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13937541842460632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032143244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14471688866615295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032157155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14084112644195557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032171270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12568853795528412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032185296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376904547214508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032199477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13175557553768158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032213731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13604329526424408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032227906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137868732213974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032241725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310232758522034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032256171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13434377312660217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032270381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363433599472046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032270382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032270382, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032352131, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.511272668838501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032352132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032352132, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1785032365796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13859963417053223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032380001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814221322536469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032394221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131597101688385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032408336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308095932006836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032422386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13376227021217346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032436697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13985638320446014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032450985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13291029632091522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032465307, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14097607135772705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032479052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14078818261623383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032493471, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13494953513145447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032507578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276204138994217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032521497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12986157834529877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032535308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13401655852794647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032549291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12761567533016205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032549292, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032549292, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032631738, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5141140818595886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032631739, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032631739, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1785032645602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225413858890533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032659353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13700087368488312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032673246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351202011108398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032687157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13181568682193756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032701321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13393452763557434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032715348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13338357210159302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032729799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13382863998413086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032744090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13241194188594818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032757988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13991568982601166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032772543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13177715241909027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032786328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12870407104492188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032799817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13097204267978668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032813335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321365386247635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032827001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13538098335266113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032827002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032827002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032909136, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.51784747838974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032909137, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032909137, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1785032922736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351058959960938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032936331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12295016646385193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032949960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044534623622894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032963639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302943378686905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032977212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510408997535706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785032991323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13551941514015198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033005009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13531766831874847, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033019139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12393872439861298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033033042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13102102279663086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033046598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13113214075565338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033060960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1301925778388977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033074996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325785100460052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033089166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087013363838196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033102969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13012462854385376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033102970, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033102970, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033184777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5229683518409729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033184778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033184778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1785033198815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13776618242263794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033212787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132360577583313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033226659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335616260766983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033240607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13039442896842957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033254519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12672127783298492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033268358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322118192911148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033282608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555727899074554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033296622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13279519975185394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033310752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13124214112758636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033325027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13885565102100372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033339199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1338048279285431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033352868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13748043775558472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033367213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315261870622635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033381012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13315919041633606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033381013, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033381013, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033462589, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5298786163330078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033462591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033462591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1785033476244, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.130939319729805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033489836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484759628772736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033503496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13760262727737427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033517301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13309729099273682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033530943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272576004266739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033544651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12926635146141052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033558299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12584500014781952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033571819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1239573135972023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033585711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13623493909835815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033599692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13704505562782288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033613573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13431251049041748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033628068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12679757177829742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033642047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13086266815662384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033655574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12338612973690033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033655574, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033655575, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033739613, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5392440557479858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033739614, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033739614, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1785033753876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322774440050125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033767856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13225281238555908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033782010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13344618678092957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033796173, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377457231283188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033810187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13614621758460999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033824243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13110728561878204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033838282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270628273487091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033852400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13477715849876404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033866332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12429249286651611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033880638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13005201518535614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033894782, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13237686455249786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033909188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374663889408112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033923556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1400359570980072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033937598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13601674139499664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785033937598, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785033937599, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034021138, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5526826977729797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034021139, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034021139, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1785034035814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618646562099457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034050074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101255893707275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034064113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217957317829132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034078519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388949751853943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034092896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12984976172447205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034107629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13817612826824188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034121800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134648859500885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034136052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369737982749939, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034150057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13443537056446075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034164417, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13044965267181396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034178733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14009904861450195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034193059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13442949950695038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034207198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12855258584022522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034221701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308031857013702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034221701, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034221702, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034304616, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5675315260887146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034304617, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034304617, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1785034319321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14158780872821808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034333715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321834772825241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034348699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332286298274994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034363143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303015649318695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034377996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516487181186676, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034392695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13310500979423523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034407216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13339214026927948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034421699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525016605854034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034436007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079212605953217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034450594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1321539580821991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034465078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13268084824085236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034479600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13668730854988098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034493882, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13594378530979156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034508603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.136916846036911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034508604, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034508604, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034592429, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5827209949493408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034592430, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034592430, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1785034606695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13114377856254578, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034621369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1348288357257843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034635808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13396303355693817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034650687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314782351255417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034665042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13652165234088898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034679332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12901028990745544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034694328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13328605890274048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034708388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12274497747421265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034722516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001649081707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034736605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13078559935092926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034750607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046371936798096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034764581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306118667125702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034778590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12861885130405426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034792704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027015328407288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034792705, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034792705, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034873709, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5937869548797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034873710, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034873710, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1785034887804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12152598798274994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034901823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12583449482917786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034915778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12812523543834686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034929917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280077278614044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034944400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121534883975983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034958527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12114281952381134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034973059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12921574711799622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785034987527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13119445741176605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035001778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13576669991016388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035015566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12715034186840057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035030191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1322098970413208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035044388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11683426052331924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035058584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12006130069494247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035072837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12418857961893082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035072837, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035072838, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035154811, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6064016819000244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035154812, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035154812, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1785035168945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12065713852643967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035183084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265815794467926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035197398, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271066963672638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035211534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13079816102981567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035225537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12055930495262146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035239677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378246128559113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035253628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800060212612152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035267680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12062317878007889, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035281834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12601973116397858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035296105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258808672428131, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035310120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778499186038971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035324635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13069500029087067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035338850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12595203518867493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035353062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275722086429596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035353063, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035353063, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035436591, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6183522343635559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035436592, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035436592, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1785035450273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274702101945877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035464960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11807235330343246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035479274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12086319178342819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035493783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12004774063825607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035508065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12274733185768127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035522226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12512077391147614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035536295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11555708944797516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035550609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1219552755355835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035564776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12396473437547684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035578858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1231248676776886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035592789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11453534662723541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035606922, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12355378270149231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035621045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957508534193039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035635037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382210046052933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035635037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035635038, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035719353, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6289666891098022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035719354, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035719354, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1785035733617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11582434177398682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035747919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11303749680519104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035762071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269591748714447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035776875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12107805907726288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035791127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12289875745773315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035805321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11510120332241058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035819141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11939450353384018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035833718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12158135324716568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035847769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12019775807857513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035861952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119416207075119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035876003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12077374011278152, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035889971, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12154814600944519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035904019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12477097660303116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035918226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11618328094482422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785035918226, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785035918227, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036002463, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6431187987327576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036002463, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036002464, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1785036016829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11952906847000122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036030728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088480055332184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036044559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10934103280305862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036058585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1133686900138855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036072591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893559455871582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036086659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12189705669879913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036100999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11625505983829498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036115653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12042949348688126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036129748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10284296423196793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036144337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11977023631334305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036158868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11258519440889359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036173255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11313463002443314, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036187515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907308012247086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036202098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11680957674980164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036202099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036202099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036286442, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6625610589981079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036286443, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036286443, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1785036300644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1210140585899353, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036314917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11913342773914337, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036329372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12297184020280838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036343767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683710128068924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036358197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414726078510284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036372540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11779043078422546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036387193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896362155675888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036401798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11898581683635712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036416127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082075983285904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036430496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11858968436717987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036444806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498495191335678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036458873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676366627216339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036472936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11300789564847946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036487513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12512464821338654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036487514, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036487514, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036572262, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6836523413658142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036572263, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036572263, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1785036586853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12365086376667023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036601465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11727097630500793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036615907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643064022064209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036630819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12141193449497223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036645556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1098303347826004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036660302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11116620898246765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036674385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374668776988983, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036689271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948322713375092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036703661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102768778800964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036718621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11561720818281174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036733095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12384845316410065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036747460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11477607488632202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036762137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800820171833038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036776795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11388816684484482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036776796, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036776796, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036859466, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7200950384140015, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036859467, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036859467, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1785036874115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1252536177635193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036888567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389647424221039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036903037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1161603182554245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036917636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11329453438520432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036932084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1192837581038475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036946709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11733483523130417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036961784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12156344950199127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036976739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12127597630023956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785036991366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11543899029493332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037006572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282321810722351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037021175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11626383662223816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037035699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150059923529625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037050234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12567493319511414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037065416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679857224225998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037065417, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037065417, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037147933, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7388628721237183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037147934, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037147934, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1785037162741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1227724626660347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037177631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848834902048111, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037192432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11405222862958908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037207484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.115668885409832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037222280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10863891243934631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037237141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607526242733002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037251848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1183989867568016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037266305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1117604523897171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037281037, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11014046519994736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037295465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085827425122261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037310382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1229117140173912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037325239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740908235311508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037339942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12075953930616379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037355098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11953001469373703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037355099, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037355099, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037439261, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7505598068237305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037439262, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037439262, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1785037454106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935276538133621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037468561, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184198185801506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037483103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11604278534650803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037498080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168144941329956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037512576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398884862661362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037527089, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413704812526703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037541463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12044068425893784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037555681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1147673949599266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037569920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10404534637928009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037584125, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1033811867237091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037598158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694815218448639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037612553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.113070048391819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037626832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11259378492832184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037641210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11568903923034668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037641247, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037641247, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037726120, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7586737275123596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037726121, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037726121, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1785037740547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541784763336182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037755210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753197968006134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037770072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11426887661218643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037784475, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11115315556526184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037799151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844042152166367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037814093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11726313084363937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037828761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389338970184326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037843208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354943364858627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037857954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670592427253723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037872285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163027584552765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037886565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045833885669708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037901087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436381191015244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037915209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805051773786545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037929568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557027161121368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785037929569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785037929569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038012534, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7637120485305786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038012535, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038012535, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1785038026855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793829709291458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038041247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11359303444623947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038055750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09706210345029831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038070190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10461191087961197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038084697, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328978091478348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038098969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563313215970993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038113550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08823878318071365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038127721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11005397140979767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038142070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209009051322937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038156649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11787804961204529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038171560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827374458312988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038186094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040409862995148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038201212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10020065307617188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038216235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712314397096634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038216235, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038216236, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038300111, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7669221758842468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038300112, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038300112, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1785038314683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11520344763994217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038329436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216192692518234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038343576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043621227145195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038359184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10522613674402237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038373873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949195384979248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038388730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346563905477524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038403497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071104034781456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038418411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768695175647736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038433008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817367047071457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038447975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10877540707588196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038462658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10477922856807709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038477239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066729873418808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038491755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10874922573566437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038506564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290159821510315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038506565, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038506566, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038589633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7687166929244995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038589634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038589634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1785038604395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938632488250732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038618623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09841214865446091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038632859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312470048666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038647397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572855174541473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038662084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111455410718918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038676754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241775006055832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038691140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001686081290245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038705600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10447492450475693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038720147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840605199337006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038734412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10497673600912094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038748785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11345429718494415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038762758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09524079412221909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038777886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09817755222320557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038792554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10733159631490707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038792555, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038792555, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038875469, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7698713541030884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038875470, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038875470, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1785038889658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09092394262552261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038904318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.103541798889637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038918886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019019782543182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038933804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09815351665019989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038948576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051246523857117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038963366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098945885896683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038978193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.097931407392025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785038992769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069130897521973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039007483, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507486760616302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039022052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432429611682892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039036815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721137374639511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039051236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439816117286682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039065484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09770926088094711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039080094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10600882768630981, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039080094, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039080095, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039163708, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707446813583374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039163709, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039163709, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1785039178747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10428828746080399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039193788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1040724590420723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039208545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10575000196695328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039223399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1034262627363205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039238420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09782432019710541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039253056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09978671371936798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039267883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469523072242737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039281997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0963578000664711, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039296827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10803906619548798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039311287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079862117767334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039325699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11684411764144897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039340210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420707404613495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039354520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11633242666721344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039368931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11494087427854538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039368932, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039368932, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039452301, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713516354560852, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039452302, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039452302, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1785039466468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055563598871231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039480895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578787863254547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039495018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108212411403656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039509409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10161294043064117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039523542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09189607948064804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039538068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09851089864969254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039552240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037716418504715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039566626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077549010515213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039580815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745874047279358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039595080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009221732616425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039609758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131066381931305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039624348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920955449342728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039639027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096871867775917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039653400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11183418333530426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039653401, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039653401, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039737997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718645930290222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039737998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039737998, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1785039752725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10770171880722046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039767340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160303860902786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039781748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046899110078812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039796354, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.091947041451931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039810732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448036134243011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039825056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691892355680466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039840416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11098015308380127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039855207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11106141656637192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039870276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12190274149179459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039885196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1043306291103363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039900119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962624847888947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039915164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12339212745428085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039930059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922529757022858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039945101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11050239205360413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785039945101, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785039945102, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040029341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7720608115196228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040029342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040029343, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1785040044121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364358872175217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040059094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11886557191610336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040073918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965602099895477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040089170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241957753896713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040104126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09621185064315796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040119065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11243341863155365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040134407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661821812391281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040149683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541021823883057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040165074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11554179340600967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040179949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09991499781608582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040195215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12512174248695374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040210221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10754769295454025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040224703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11401233077049255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040239263, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10751233994960785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040239264, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040239264, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040322945, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7724158763885498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040322946, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040322946, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1785040337859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11352979391813278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040352815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11540025472640991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040367797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11075736582279205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040382936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11796253174543381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040397858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11442281305789948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040412931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11890258640050888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040427933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12722250819206238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040442525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12131829559803009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040457122, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159488707780838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040472058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033650487661362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040486640, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867641121149063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040501577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1079489141702652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040516452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11459318548440933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040531380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409846693277359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040531381, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040531381, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040613686, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726709246635437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040613687, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040613687, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1785040628666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11992654949426651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040643611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10680367052555084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040658356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1075059026479721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040673635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851582139730453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040688610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11087138950824738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040703335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10935825109481812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040717969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362570524215698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040732507, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085314229130745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040747052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10509642958641052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040761112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10050130635499954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040775632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441181063652039, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040789972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09565611928701401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040804285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10380551964044571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040818378, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10040654987096786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040818379, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040818379, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040899265, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7728919982910156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040899266, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040899266, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1785040913302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913611203432083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040927576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093310475349426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040941759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09637162834405899, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040955934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10824834555387497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040970006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498510837554932, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040984071, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500358790159225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785040998134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393832623958588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041012460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427729040384293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041026659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10126793384552002, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041041061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11964753270149231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041055570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11688214540481567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041069832, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10090741515159607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041084588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10002478212118149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041098830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12057120352983475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041098830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041098831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041180802, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773169994354248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041180803, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041180803, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1785041195484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106390431523323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041210053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965723991394043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041224620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10462447255849838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041239265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304269939661026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041253530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814250260591507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041268233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10429040342569351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041282613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10609500110149384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041296980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241357028484344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041311684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.105167917907238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041326272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1080334484577179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041340784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570752620697021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041355146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11454000324010849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041369425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068277508020401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041383911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10245957225561142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041383912, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041383912, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041466914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7733433842658997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041466915, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041466915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1785041481323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952801465988159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041495857, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09312146157026291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041510088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09609147161245346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041524579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671977162361145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041538883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0965789407491684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041553396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09441717714071274, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041567506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10035320371389389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041581719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0968392863869667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041596273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285723209381104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041610709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777643322944641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041625451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10465335100889206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041639740, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324513167142868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041653970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11409050226211548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041668768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095251142978668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041668768, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041668768, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041751828, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734858393669128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041751829, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041751829, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1785041766304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09801874309778214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041780711, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683964937925339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041795369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954195261001587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041809586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10132191330194473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041824186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09924669563770294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041838791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10748124867677689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041853002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046445369720459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041867304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593511909246445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041881912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268934816122055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041896055, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10079678893089294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041910187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995676577091217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041924312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1028614491224289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041938524, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907084494829178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041952610, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481061577796936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785041952611, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785041952612, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042035116, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.773608922958374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042035117, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042035117, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1785042049369, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956225544214249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042063699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10667037963867188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042077953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09087102115154266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042092076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102563351392746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042106455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645650327205658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042120572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736025124788284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042134669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1103699579834938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042148918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228993743658066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042163359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057242214679718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042177721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10384442657232285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042192325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243874788284302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042206618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10720664262771606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042221127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424961149692535, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042235747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108835868537426, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042235748, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042235748, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042318631, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737111449241638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042318632, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042318633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1785042333110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328163206577301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042347597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10285752266645432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042361699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355991870164871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042375639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260442644357681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042390657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725440829992294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042404935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10070332139730453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042419384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09810589253902435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042434044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10154598951339722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042448688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119617372751236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042463262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526324808597565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042477707, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10395409166812897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042492115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10212095081806183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042506614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362622141838074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042521025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653871297836304, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042521026, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042521026, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042604714, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738252282142639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042604715, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042604715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1785042619183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855220258235931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042633908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10169089585542679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042648409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344725847244263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042663064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100530833005905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042677847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10441668331623077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042692140, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11320183426141739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042706722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.101235531270504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042721022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11033476144075394, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042735755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254958808422089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042750842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692405700683594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042765522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031406968832016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042780321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557381063699722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042795312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052471324801445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042810095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09806354343891144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042810095, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042810096, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042894983, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738267183303833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042894984, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042894984, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1785042909611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102642327547073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042924249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898138791322708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042938869, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635606199502945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042953059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10062753409147263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042968149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842334479093552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042982401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1093006283044815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785042996885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10780488699674606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043011505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155586689710617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043025807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373691260814667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043039976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10044945776462555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043054021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368963330984116, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043068242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10301890969276428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043082468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053866595029831, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043096638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729727894067764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043096639, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043096639, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043179486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739631533622742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043179487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043179487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1785043193825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10133937746286392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043208044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001853853464127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043222603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092868372797966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043236946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453467071056366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043251182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09393051266670227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043266023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297674477100372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043280324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011042594909668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043294737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772378742694855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043309389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331428796052933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043324343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351146757602692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043338793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929463803768158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043353482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11420144140720367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043368268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10538181662559509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043383120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278709977865219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043383120, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043383121, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043465312, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740310430526733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043465313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043465313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1785043480449, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263345927000046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043495182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10881412774324417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043509817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10947076231241226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043524675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666826367378235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043538999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10163544863462448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043553317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737879574298859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043568652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11876723170280457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043583720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094549223780632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043598611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11590006947517395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043613784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12140027433633804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043628604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10704666376113892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043643454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985877364873886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043658436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11530294269323349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043673281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11024171113967896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043673282, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043673282, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043755753, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740340232849121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043755754, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043755754, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1785043770727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11318147927522659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043785837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940331965684891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043800821, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039263755083084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043815717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643842071294785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043830476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10427824407815933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043845302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11373378336429596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043860154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10698573291301727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043875064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11472335457801819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043890233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11275634914636612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043905520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11935865879058838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043920742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607051849365234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043935702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1175990030169487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043950765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685086250305176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043966158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1184300109744072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785043966159, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785043966159, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044048683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742920517921448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044048684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044048684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1785044063797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11878899484872818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044078688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10795008391141891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044093538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11934702098369598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044108280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11414128541946411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044123144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10783306509256363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044138438, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11552131921052933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044153092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097051203250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044168019, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11691984534263611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044182952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196579039096832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044197547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10632389038801193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044212143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11237234622240067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044226908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058461144566536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044241522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11700145900249481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044256443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485583543777466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044256443, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044256444, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044338341, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742561101913452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044338342, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044338342, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1785044352936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10973937809467316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044368161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11991646885871887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044382913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997551679611206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044397587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11347004771232605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044412189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11622529476881027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044426825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10892128199338913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044441649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812753438949585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044456537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10983459651470184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044471494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11196254938840866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044486147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11197934299707413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044500980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11011625081300735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044516006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11085846275091171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044530800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11137691140174866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044545450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10916116833686829, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044545451, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044545451, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044626476, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743262648582458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044626476, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044626477, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1785044640993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11793509125709534, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044656041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053190380334854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044670918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074178665876389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044685748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360893607139587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044700737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052609235048294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044715830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131493747234344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044730594, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781487822532654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044745283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707570612430573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044760167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09734514355659485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044774885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167429596185684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044789545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517582297325134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044804146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10424518585205078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044819022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980686545372009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044833947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11589615046977997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044833948, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044833948, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044918022, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7742754817008972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044918023, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044918023, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1785044933091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11438882350921631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044947706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10464029014110565, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044962267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10026340186595917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044976868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0919642373919487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785044991401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498019844293594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045005858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358390212059021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045020534, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10232198983430862, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045035407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265844106674194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045049975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731655359268188, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045064623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10286223888397217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045079609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10907576233148575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045094584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042637377977371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045109673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766849666833878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045124433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10703017562627792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045124434, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045124434, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045209008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743895649909973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045209009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045209009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1785045223603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918770730495453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045238310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10268652439117432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045252720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081671714782715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045267134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09880869090557098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045281983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11674102395772934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045296521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984823107719421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045310854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10337348282337189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045325651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1086292490363121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045340252, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10812889784574509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045354489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836725682020187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045368853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010424792766571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045383177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09953165054321289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045397283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587770491838455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045411464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08756517618894577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045411465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045411465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045495064, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745414972305298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045495065, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045495065, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1785045509380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551838040351868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045523694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11148380488157272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045538021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10702306032180786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045552280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10111048072576523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045566589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10223418474197388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045581024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09079927951097488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045595360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10354651510715485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045609644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09707368165254593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045623774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10351338982582092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045638200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628857463598251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045652718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100715771317482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045667181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10840624570846558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045681404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603771358728409, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045695880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288653522729874, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045695880, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045695880, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045780895, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743710279464722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045780896, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045780896, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1785045795476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09605874121189117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045810345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508861392736435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045824727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09490539133548737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045839168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10194901376962662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045853570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09806695580482483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045867598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038144141435623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045881666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933251678943634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045896884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176768898963928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045911362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088237389922142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045925900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10957042872905731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045940698, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10645316541194916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045955499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449289530515671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045970145, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913780331611633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045984884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10896828025579453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785045984885, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785045984885, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046068386, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745290398597717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046068387, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046068388, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1785046082830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608398169279099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046097319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10411304980516434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046111664, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280042886734009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046126234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10029977560043335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046140897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10656514018774033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046155619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09558530896902084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046170824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10484892129898071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046185425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274410247802734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046200201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041475832462311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046215210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09869467467069626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046229651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09782709926366806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046244044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183434188365936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046258666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09032073616981506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046273350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694222897291183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046273351, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046273352, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046356288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7745841145515442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046356289, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046356289, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1785046370977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09841860085725784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046385911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089950263500214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046400469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09906332939863205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046415414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976993292570114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046430465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107507556676865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046445327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246255248785019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046459854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209770500659943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046474434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544847697019577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046489358, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10150986909866333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046503812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144793778657913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046518805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10118843615055084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046533136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853132605552673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046547760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508773475885391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046562501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1057133674621582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046562501, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046562502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046645740, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7747053503990173, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046645741, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046645742, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1785046660853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628858953714371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046675798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09800248593091965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046690403, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716985166072845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046705673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611335933208466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046720653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10201172530651093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046735298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10752817243337631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046750209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738098084926605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046765345, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09806769341230392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046780283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529821366071701, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046795081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443245619535446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046809897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255927592515945, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046824814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10970936715602875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046839718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10648257285356522, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046854627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962116926908493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046854628, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046854628, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046940784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749870419502258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046940784, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046940785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1785046955150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288669914007187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046969588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498549789190292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046983995, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09434322267770767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785046998877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827584028244019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047013339, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10238650441169739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047028547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09989573806524277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047043522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662516206502914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047058532, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11013785749673843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047073399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1095050573348999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047088402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09905343502759933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047103269, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219220072031021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047117718, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0985604077577591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047132712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241482198238373, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047147323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11052330583333969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047147324, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047147324, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047231474, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7750868797302246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047231475, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047231475, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1785047245871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10570449382066727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047261192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467128455638885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047275703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590463876724243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047290558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134190112352371, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047305787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662292689085007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047320873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108212910592556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047335572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1077498346567154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047350371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09757579118013382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047365223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1063499003648758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047379749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329656302928925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047394488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844287276268005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047408702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339990258216858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047423499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10662691295146942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047438148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09477879852056503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047438184, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047438185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047522788, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7751519680023193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047522789, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047522789, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1785047537545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09955647587776184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047552255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10568113625049591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047566912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10313506424427032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047581688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448441654443741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047596314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10826760530471802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047610886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1224060133099556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047625601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10769456624984741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047640295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001680999994278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047655272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11101870238780975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047670141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1108303964138031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047685261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11509428173303604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047700353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09788370877504349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047715555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10700041800737381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047730651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385849326848984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047730689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047730689, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047814821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7755433917045593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047814822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047814822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1785047829990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09667797386646271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047845100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09739083051681519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047859951, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11255459487438202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047875048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11756789684295654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047889755, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354352533817291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047904198, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11086272448301315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047919424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10753082484006882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047934185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11995879560709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047949062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482539981603622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047964365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108317069709301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047979484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018221080303192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785047994373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11068209260702133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048009275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11335147172212601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048024092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114297062158585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048024093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048024093, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048107482, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7758362889289856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048107483, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048107483, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1785048122274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728780925273895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048137093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11035313457250595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048151993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418145895004272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048166825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11079065501689911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048181632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810791701078415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048196662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10789129137992859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048211467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148188471794128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048226188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229000985622406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048241065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12109246104955673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048255942, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938509553670883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048270729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10893795639276505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048285862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383409261703491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048300965, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1036749854683876, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048316110, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10910143703222275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048316111, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048316112, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048399056, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.776186466217041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048399057, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048399057, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1785048413972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11382982134819031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048429137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11842318624258041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048444245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11362925916910172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048459556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11238491535186768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048474854, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11141747981309891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048489840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11946381628513336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048505049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11166662722826004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048519634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10956371575593948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048534975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.110596664249897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048550058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09923901408910751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048565246, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980060696601868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048580947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10990498960018158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048596387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10394967347383499, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048611435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11023806780576706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048611435, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048611436, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048694846, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7764796614646912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048694847, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048694847, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1785048709874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11327142268419266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048725288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12793390452861786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048740459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809746384620667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048755500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250756680965424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048770772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272880434989929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048786102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111846923828125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048801562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11398662626743317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048817214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11274465173482895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048832502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10670453310012817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048847797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796744376420975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048863265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12988406419754028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048878392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11296815425157547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048894167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111888587474823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048909955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601226776838303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785048909955, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785048909955, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785048995517, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7767296433448792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785048995518, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785048995518, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1785049010824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09579312801361084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049026249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11138460785150528, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049041531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1219002902507782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049056886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699749737977982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049072242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10955600440502167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049087453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10716854780912399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049102584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11220987886190414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049117456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355932056903839, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049132512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331756621599197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049147012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717823147773743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049161812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688144713640213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049176818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11131212115287781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049191920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09220009297132492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049206624, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10349095612764359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049206625, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049206625, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049292541, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768514156341553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049292542, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049292542, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1785049307613, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0998007282614708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049322437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686541557312012, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049337094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140117198228836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049351661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10749848186969757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049366277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963109880685806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049381395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403084754943848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049396186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10762524604797363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049411026, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283259093761444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049425833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997913241386414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049440868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09469850361347198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049455661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09707077592611313, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049470305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09686420857906342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049484743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026613637804985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049499533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10057331621646881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049499533, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049499534, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049586159, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768312692642212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049586160, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049586160, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1785049601015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10324925929307938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049616162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10270736366510391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049631206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393742471933365, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049646099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10633566975593567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049660973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189548134803772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049676266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09964390844106674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049691777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855940729379654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049706798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530519485473633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049721891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673359781503677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049736881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601437091827393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049752106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386841744184494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049767012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776780545711517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049781725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615330934524536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049797260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1129695251584053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049797305, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049797306, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049882926, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773341536521912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049882927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049882927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1785049897748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889171063899994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049912741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11163238435983658, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049928084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.106564961373806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049943607, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10684159398078918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049958709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10035856813192368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049973825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152994096279144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785049988983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09582436829805374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050003783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10678065568208694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050018422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09937532246112823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050033065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10184117406606674, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050047989, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1010720282793045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050062550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255014151334763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050077072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09922264516353607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050091967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393715649843216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050092005, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050092006, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050176862, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775543928146362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050176863, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050176863, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1785050191848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09687646478414536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050206822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761061310768127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050221585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046671330928802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050236348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279783606529236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050251021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09440979361534119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050265964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696648061275482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050281094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10319008678197861, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050295919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10330111533403397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050310996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10446522384881973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050325820, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180733144283295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050340876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0953916683793068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050355937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110544949769974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050370831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172449588775635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050385419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015208512544632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050385460, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050385461, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050469473, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779313325881958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050469474, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050469474, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1785050484028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827622771263123, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050498807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728023946285248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050513527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151971131563187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050527913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513811558485031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050542972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10058874636888504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050557658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084088534116745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050572282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09988905489444733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050587233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09811810404062271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050602150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10879916697740555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050616970, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088314801454544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050631715, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989164263010025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050646701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09392233192920685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050661309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09429013729095459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050676021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10553766787052155, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050676022, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050676022, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050758008, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7785099148750305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050758009, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050758009, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1785050772772, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024998128414154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050787435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11365505307912827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050802149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10017678886651993, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050816871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10524698346853256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050831551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1099468246102333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050846223, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09997612982988358, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050861020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954024642705917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050875815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045772045850754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050890647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10638388246297836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050905455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10018035024404526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050920444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09727338701486588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050935732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207836121320724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050950952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11841566115617752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050966036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894139111042023, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785050966037, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785050966037, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051047997, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7788505554199219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051047998, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051047998, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1785051063075, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11437927186489105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051077758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10107389092445374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051092873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09956853836774826, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051107858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007462665438652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051122937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448732227087021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051137885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09894277155399323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051152716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002778485417366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051167763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058330088853836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051182441, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055213212966919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051197006, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09330917149782181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051212481, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10352843254804611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051227020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10142980515956879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051241694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09418658912181854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051256537, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09950333833694458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051256538, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051256538, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051338575, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7790018320083618, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051338576, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051338576, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1785051353446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11102540791034698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051368203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11338824033737183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051382735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082482546567917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051397487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10664589703083038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051412189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10243093222379684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051426888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1017770767211914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051441395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10608028620481491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051455954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1087418645620346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051470600, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09756886214017868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051485123, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09698090702295303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051499777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002243310213089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051514097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09661427140235901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051528991, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10210713744163513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051543515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10230713337659836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051543516, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051543516, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051625251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7791846394538879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051625252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051625252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1785051639811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10250681638717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051654247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10810030996799469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051668812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11193227767944336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051684031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09294495731592178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051698905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1024547666311264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051713690, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744946449995041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051728570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1166396290063858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051743262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10088343173265457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051758158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591216385364532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051773482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895474255084991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051788374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09750571846961975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051803176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10714153945446014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051818121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09842141717672348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051832626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456285625696182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051832627, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051832627, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051915149, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796854972839355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051915149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051915150, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1785051930445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11010333150625229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051945154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358525067567825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051959533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09176407009363174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051974796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055691003799438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785051989550, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10792253166437149, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052004285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1115415021777153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052019615, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09655426442623138, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052034734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341567546129227, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052049709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050177812576294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052064570, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0982932522892952, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052079440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344594717025757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052094305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10467391461133957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052109011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002132296562195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052123948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11330369114875793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052123949, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052123949, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052206263, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7796527147293091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052206264, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052206264, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1785052221002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859814286231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052236206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471327602863312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052251039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10199669003486633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052266068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10835345089435577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052280873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10208049416542053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052295933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971112921833992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052310633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10241781175136566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052325284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11113369464874268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052340061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10125716030597687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052354979, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09447497874498367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052369792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378646105527878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052385002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191640794277191, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052400265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137928277254105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052415184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11784270405769348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052415185, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052415185, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052500985, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7799276113510132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052500986, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052500986, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1785052516197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981628507375717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052531111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09699826687574387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052546280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10601243376731873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052561547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250147223472595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052576939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11696992069482803, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052592157, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09578622877597809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052607332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636766254901886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052622162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103476792573929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052637197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011614054441452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052652161, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10438644886016846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052666637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10658632963895798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052682105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948285460472107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052696783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09938912093639374, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052711605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11342119425535202, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052711606, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052711606, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052797095, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7804022431373596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052797096, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052797096, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1785052812210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216588318347931, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052827499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11124176532030106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052842466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635153204202652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052857372, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11885037273168564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052872228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11619836091995239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052887336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10034791380167007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052902435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1174008771777153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052917299, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389193683862686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052932294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618581622838974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052947274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10689610242843628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052962408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10721380263566971, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052977401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10526325553655624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785052992582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710801929235458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053007266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11558616161346436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053007267, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053007267, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053088519, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7807079553604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053088520, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053088520, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1785053103803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11172465234994888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053118720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371177643537521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053134052, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10229676961898804, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053149245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11612074822187424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053164473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793616622686386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053179308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10489306598901749, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053194335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386357456445694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053209300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132175549864769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053224686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11778215318918228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053239914, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10482483357191086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1785053255242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11564325541257858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053270432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11180909723043442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053285057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10782753676176071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053300181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11619686335325241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053300182, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053300182, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053381695, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810214161872864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053381696, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053381697, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1785053396502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121721029281616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053411847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11133447289466858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053427057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11527463048696518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053442041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014904290437698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053457445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09854444861412048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053472804, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10742687433958054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053488133, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902872145175934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053503375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819010436534882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053518678, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10012488812208176, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053533769, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10582534223794937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053548823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11474596709012985, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053563746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10989334434270859, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053578552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11750618368387222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053593801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11215201020240784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053593802, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053593802, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053677228, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814888954162598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053677229, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053677229, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1785053692146, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11149968206882477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053707276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562536865472794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053722119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056513339281082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053737181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11595930904150009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053752343, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11221322417259216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053767611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11652375012636185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053783214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898607432842255, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053798673, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10843845456838608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053814186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10797138512134552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053829859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1082003191113472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053845601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11769521236419678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053861051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044465005397797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053876359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1030433177947998, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053891831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11027389764785767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785053891831, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053891832, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053972861, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7811171412467957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053972861, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053972862, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1785053987668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10567289590835571, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054002908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11056801676750183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054017827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11880995333194733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054033191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628723353147507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054047768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490485280752182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054062527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10072874277830124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054077996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620785504579544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054092760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10924496501684189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054107686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1120743602514267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054122362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09363242983818054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054137340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679542809724808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054152163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09952055662870407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054166486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09829719364643097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054180880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708057135343552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054180880, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054180881, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054263206, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7810669541358948, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054263207, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054263207, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1785054277915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007923111319542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054292961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09290392696857452, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054307892, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0963250994682312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054322630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868389159440994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054337303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10776635259389877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054352004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11241219192743301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054366499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09500519931316376, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054381266, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10521692782640457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054395941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10602521896362305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054410504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10996668040752411, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054424984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09757247567176819, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054440011, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09828417748212814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054454850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11128991842269897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054469351, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09865450114011765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054469352, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054469352, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054551501, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7816809415817261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054551501, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054551502, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1785054566440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10561046004295349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054581169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0955706313252449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054596282, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10295005887746811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054611344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09557194262742996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054626291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10189777612686157, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054641048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11073702573776245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054655980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10646399855613708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054670576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09520167857408524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054685370, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11325940489768982, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054700106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10397934913635254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054714480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344891250133514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054729876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156407207250595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054744694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10304336994886398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054759937, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1012105718255043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054759938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054759938, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054842772, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7814342975616455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054842773, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054842773, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1785054858170, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10042937844991684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054873489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09895973652601242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054888592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10099714249372482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054903288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10171964019536972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054918416, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09881605207920074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054933213, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10045759379863739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054947797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09683693945407867, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054962390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10009517520666122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054977334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1032460406422615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785054992097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10814335197210312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055007051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348436236381531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055021986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1073966696858406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055036818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10138839483261108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055051340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0956558957695961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055051341, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055051341, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055135528, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7818154692649841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055135529, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055135529, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1785055150557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09400293231010437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055165098, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0990832969546318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055179779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09640584886074066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055194485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09961166232824326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055209304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10393788665533066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055224846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09508147090673447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055239993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10174640268087387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055255321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0974823534488678, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055270086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0976036936044693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055285195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1038234606385231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055300365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0934252068400383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055315917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723937302827835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055331332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433796793222427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055346895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809801518917084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055346896, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055346897, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055430313, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7819897532463074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055430313, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055430313, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1785055445533, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047096997499466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055460500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10112645477056503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055475444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10535863786935806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055491004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291267186403275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055505816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09605276584625244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055520746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10469988733530045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055535743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09460142254829407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055550396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11387782543897629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055565000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824220836162567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055580377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08957105875015259, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055595422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10011179745197296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055610335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10408267378807068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055625129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10433867573738098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055639902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841730237007141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055639903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055639903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055722196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7822414040565491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055722197, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055722197, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1785055736908, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962751716375351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055751320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051267609000206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055766119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09754883497953415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055781087, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0999884307384491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055796002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10014534741640091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055810787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09425465017557144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055825540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10403574258089066, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055840663, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10226717591285706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055855222, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019170492887497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055870253, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10997647792100906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055884608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09742698818445206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055899484, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936152398586273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055914249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09984339773654938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055929051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510250926017761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785055929051, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785055929052, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056011426, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.782217800617218, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056011427, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056011428, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1785056026461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10100886970758438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056041348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09520447254180908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056056327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09824647754430771, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056071099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760688781738281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056086216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10278007388114929, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056101212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08729714900255203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056116296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573576390743256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056131598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09288742393255234, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056146768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426854342222214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056161747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09526742249727249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056176661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09462322294712067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056191444, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10253601521253586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056206270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11127454787492752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056220990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0977720245718956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056220991, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056220991, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056305251, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7828539609909058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056305252, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056305252, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1785056319591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09515446424484253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220364800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056334916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09562563896179199, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220528640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056349621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027037560939789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220692480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056364482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1021590456366539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220856320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056379795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10370168834924698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221020160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056395132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10564199835062027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221184000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056410322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10796640813350677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221347840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056425259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1155538260936737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221511680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056440312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808967798948288, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221675520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056455247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09902267158031464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 221839360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056470201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11042363196611404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222003200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056485267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09637783467769623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222167040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056500393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09421882778406143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222330880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056515336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838200896978378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 222494720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1785056515337, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} +:::MLLOG {"namespace": "", "time_ms": 1785056515337, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 222494720, "epoch_num": 0.09712383428602152}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed914924190.log b/recommendation_v4/rcp_logs/gbs_8192/seed914924190.log new file mode 100644 index 000000000..b79d012e0 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed914924190.log @@ -0,0 +1,1840 @@ +:::MLLOG {"namespace": "", "time_ms": 1784651468375, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784651468375, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784651489971, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784651489973, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "seed", "value": 914924190, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784651489974, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784651489975, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784652395978, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784652395979, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784652396394, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784652514109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13951851427555084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784652526454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13819362223148346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652539226, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855282962322235, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652552094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13848203420639038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652565078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853426277637482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652578220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834579288959503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652591132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13838481903076172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652604455, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13959240913391113, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652617945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387408822774887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652630906, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853862881660461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652644464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138670414686203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652658115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13876578211784363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784652671643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13840147852897644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652685001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907094299793243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652685001, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652685002, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652767451, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49707460403442383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652767455, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652767455, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784652780579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13830280303955078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652793902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382291615009308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652807796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13837756216526031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652821085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382341980934143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652834776, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13795702159404755, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652848652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13793408870697021, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652861891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796277344226837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652875274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13797645270824432, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652889203, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375451534986496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652902713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13801603019237518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652916696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383693814277649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652930568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13861341774463654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652944117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783757388591766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652958210, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842548429965973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784652958210, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784652958211, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653039161, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4972328543663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653039162, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653039162, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784653052639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376173496246338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653066356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13813334703445435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653080115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730737566947937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653094218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13645467162132263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653107775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376720815896988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653121155, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13774965703487396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653134627, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13702039420604706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653149106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690844178199768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653162963, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13653355836868286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653176566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739821314811707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653190327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374027132987976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653203757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13751189410686493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653217962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13762672245502472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653231679, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13745591044425964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653231680, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653231680, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653312642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49742311239242554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653312644, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653312644, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784653325849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662531971931458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653339468, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13741935789585114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653352899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13649596273899078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653367178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13707399368286133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653381060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736817240715027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653395027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375395506620407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653408988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13647836446762085, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653422778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13736170530319214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653437044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13672098517417908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653450928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13872510194778442, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653464682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361525058746338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653478306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378493309020996, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653492111, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13510721921920776, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653506038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1363954097032547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653506038, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653506039, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653591716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49773159623146057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653591717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653591717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784653605341, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1360086351633072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653618974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525965809822083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653632317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730329275131226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653646025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13433951139450073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653659954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644443452358246, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653673629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319307565689087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653687562, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135467991232872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653701028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13519659638404846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653714861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777241110801697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653728318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13521729409694672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653741935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14829328656196594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653756072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13540995121002197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653769677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372507691383362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653783567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13771967589855194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653783568, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653783569, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653869073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4981797933578491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653869074, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653869074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784653882494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352488398551941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653895990, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13994455337524414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653909656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13592812418937683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653923120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13297617435455322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653936548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13474617898464203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653950194, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13929271697998047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653963790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330358266830444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653977706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285589218139648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784653991453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330314576625824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654004948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13133813440799713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654018632, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308990865945816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654032014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13493920862674713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654045648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13178613781929016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654059805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13907888531684875, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654059806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654059807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654144641, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.4987562894821167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654144642, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654144642, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784654158399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13369795680046082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654172080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326720893383026, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654185946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13571718335151672, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654199677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512013852596283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654213421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770067691802979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654227258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12750589847564697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654240840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13557960093021393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654254691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12935368716716766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654268702, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356485962867737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654282890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13266897201538086, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654296834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13361451029777527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654311160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13436509668827057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654325504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13534866273403168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654339540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133548304438591, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654339541, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654339541, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654423829, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.49961382150650024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654423830, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654423830, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784654438007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254708051681519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654452028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381817936897278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654466209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13550618290901184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654480701, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355738341808319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654494867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13960345089435577, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654508747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332981437444687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654523681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13525442779064178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654537895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347357511520386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654552206, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13290861248970032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654566691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13195425271987915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654580819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596037030220032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654595420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13352897763252258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654609585, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348065316677094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654623572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13054174184799194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654623573, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654623573, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654708780, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5006323456764221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654708781, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654708781, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784654723434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137940913438797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654737741, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13502950966358185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654752062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13562911748886108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654766478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13117176294326782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654780779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13373509049415588, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654795180, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13419216871261597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654809700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369687020778656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654824149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1340666115283966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784654838717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329977959394455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654852839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12979090213775635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654866736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12769101560115814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654880730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13306543231010437, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654894684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12949402630329132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654908298, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247875988483429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784654908299, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654908299, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654990145, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5017739534378052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654990146, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784654990146, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784655004274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13116785883903503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655018152, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13609525561332703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655032350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13326500356197357, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655046541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13049113750457764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655060221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.128273606300354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655074275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12674881517887115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655088504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12892068922519684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655102185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13235504925251007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655115896, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353323757648468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655129746, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1250692903995514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655143427, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12883499264717102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655157022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.134393572807312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655170905, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13106901943683624, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655184939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292581856250763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655184939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655184940, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655268887, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032806992530823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655268889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655268889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784655282870, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070078194141388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655296643, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924572825431824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655310101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318179547786713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655324591, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12379670888185501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655338945, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12948788702487946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655352778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1259354203939438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655366725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13016308844089508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655380858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196028232574463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655394742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561828434467316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655409018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13723546266555786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655422964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12375228852033615, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655437360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12330742925405502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655451557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12565001845359802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655465587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12902140617370605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655465588, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655465589, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655549166, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5050154328346252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655549167, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655549167, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784655562695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13642561435699463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655577108, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13656863570213318, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655591100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356521099805832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655604911, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13999302685260773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655618737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13662689924240112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655632576, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827961683273315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655646514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12740936875343323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655660365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12622682750225067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655674207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14209617674350739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655688467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1411171853542328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655702777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12726716697216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655716877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12800811231136322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655730961, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326916515827179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655744445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158592581748962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655744445, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655744446, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655828456, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5067891478538513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655828457, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655828457, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784655843270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1414201706647873, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655857653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14239457249641418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655871807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13537077605724335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655886187, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13938970863819122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655900675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1451673060655594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655914840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409745216369629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655929371, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258518397808075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655943781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13755619525909424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655958409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13162946701049805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655972969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13603952527046204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784655987419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13777753710746765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656001677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13304005563259125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656016506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13446283340454102, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656030867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13638289272785187, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656030868, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656030869, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656113001, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5086406469345093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656113002, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656113002, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784656127008, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13863995671272278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656141460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789090514183044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656155871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13182172179222107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656170265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13295626640319824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656184681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.133761465549469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656199360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13982853293418884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656213929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300755620002747, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656228623, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14127117395401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656242681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14079280197620392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656257362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513807952404022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656271642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277136653661728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656285700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006626069545746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656299703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13412229716777802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656313855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12770019471645355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656313856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656313857, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656395777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5110849142074585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656395779, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656395779, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784656409891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13217739760875702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656423861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691164553165436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656438177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13357456028461456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656452406, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13202334940433502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656466868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1339401751756668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656481080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13358932733535767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656495703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13377182185649872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656510135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326957494020462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656524275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13982050120830536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656539148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131998211145401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656553361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880538403987885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656567114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108496367931366, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656581002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242831826210022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656595002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355794370174408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656595002, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656595003, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656676633, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5145903825759888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656676634, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656676634, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784656690581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13351312279701233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656704456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12299859523773193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656718258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13058851659297943, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656732131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304146647453308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656745992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13513275980949402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656760329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353587955236435, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656774254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355082094669342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656788668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12419579923152924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656802778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309286504983902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656816521, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310889571905136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656831204, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13035368919372559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656845288, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332540512084961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656859450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1308698207139969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656873385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13001573085784912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656873386, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656873387, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656955088, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5192397236824036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656955089, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656955090, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784656969163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802136480808258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656983279, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1323295682668686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784656997482, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335235834121704, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657011877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13055014610290527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657026117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12699496746063232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657040456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1320924162864685, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657055320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13559795916080475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657070045, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13317333161830902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657084593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149167597293854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657099375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1388409584760666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657113941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13381518423557281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657127887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375548392534256, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657142653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13193529844284058, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657156734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1334007829427719, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657156735, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657156735, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657239924, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5258198380470276, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657239925, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657239925, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784657253853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13107047975063324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657267668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13486658036708832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657281593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377488523721695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657295636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322030007839203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657309549, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1275329887866974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657323535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918399274349213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657337450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12603016197681427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657351331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12419457733631134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657365457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13616912066936493, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657379712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13730286061763763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657393943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13463373482227325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657408781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269654929637909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657423220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13101130723953247, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657436859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1235862448811531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657436859, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657436860, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657520321, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5354706645011902, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657520322, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657520323, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784657534773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242857158184052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657548875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13244673609733582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657563267, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374757766723633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657577625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1377996802330017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657591737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13621146976947784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657606029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13132013380527496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657620366, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1272916942834854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657634868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13484273850917816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657649065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12455197423696518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657663544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1304454505443573, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657677978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13234888017177582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657692485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13406121730804443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657706985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14014868438243866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657721214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13618884980678558, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657721214, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657721215, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657801888, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5501266717910767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657801889, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657801889, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784657816867, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13665083050727844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657831220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1314096450805664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657845259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324666440486908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657859667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1387152075767517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657874588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12996986508369446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657889684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382867395877838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657904303, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347719132900238, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657919053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369338184595108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657933827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13458847999572754, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657948682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13061945140361786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657963432, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1402703821659088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657978150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13450050354003906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784657993010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12880362570285797, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658008014, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13322190940380096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658008014, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658008015, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658089452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5667417645454407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658089454, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658089454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784658104334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14171868562698364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658119153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13259850442409515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658134488, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13327069580554962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658149519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13065870106220245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658164609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511024415493011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658179556, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1330936849117279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658194053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383358716964722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658208687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351194828748703, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658223218, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1310320347547531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658237915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1324935406446457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658252590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1326628178358078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658267176, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1366458386182785, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658281387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1359609216451645, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658296024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13686227798461914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658296025, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658296025, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658377506, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5840389132499695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658377507, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658377507, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784658392056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13118776679039001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658406879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13469569385051727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658421402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13356541097164154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658436377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315343827009201, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658450781, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13651368021965027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658465109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12919391691684723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658480038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13319405913352966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658494328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12312334775924683, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658508830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12993106245994568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658523335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13087566196918488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658537841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13006234169006348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658552281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046598434448242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658566810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12808293104171753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658581272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12985174357891083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658581273, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658581273, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658661563, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.597716748714447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658661563, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658661564, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784658675716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12123583257198334, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658690165, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1262061446905136, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658704548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1277758926153183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658719012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13236358761787415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658733901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306033879518509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658748566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12086113542318344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658763407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294410079717636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658778337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1306142508983612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658793228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13561289012432098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658807391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1269514560699463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658822520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131390780210495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658836842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11679404228925705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658851214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983849853277206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658865656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123848095536232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658865657, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658865657, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658945977, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6089483499526978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658945979, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658945979, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784658960297, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12016788125038147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658974545, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1258222609758377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784658989113, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1270771324634552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659003700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13002334535121918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659017865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11932457983493805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659032074, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13354304432868958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659046171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12764734029769897, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659060260, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11971744894981384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659074727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553542852401733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659089289, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12532325088977814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659103497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615294963121414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659118254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1298445761203766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659132817, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12553173303604126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659147335, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1274186074733734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659147336, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659147336, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659228632, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6194322109222412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659228633, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659228633, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784659242604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1253458857536316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659257606, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11656416207551956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659272162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12034222483634949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659286461, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11922803521156311, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659300672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12289102375507355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659314888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12420984357595444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659329005, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11406473815441132, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659343062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12138283252716064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659357137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12331665307283401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659371186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1205926388502121, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659385077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11343570053577423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659399102, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11981081962585449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659413086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10712503641843796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659427320, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1221344992518425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659427320, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659427321, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659510133, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.631496250629425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659510134, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659510134, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784659524495, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11436984688043594, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659539004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210490018129349, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659553143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12659451365470886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659568058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896269023418427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659582426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12106858938932419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659596894, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11364932358264923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659610835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667665839195251, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659625988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11942015588283539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659640408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1187303438782692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659654895, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11067013442516327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659669396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11761705577373505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659683800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11983279138803482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659698265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12262414395809174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659712806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11638651043176651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659712806, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659712807, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659795585, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.654762864112854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659795586, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659795586, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784659809874, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11594486981630325, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659824129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10968119651079178, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659838316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10740665346384048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659852620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154486238956451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659867053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10652945190668106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659881723, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12189936637878418, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659896433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11418909579515457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659911409, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11817489564418793, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659926084, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09919141232967377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659941150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11925877630710602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659956147, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110080033540726, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659971228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11235302686691284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784659986029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793550312519073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660000805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593995243310928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660000805, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660000806, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660083148, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6963437795639038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660083149, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660083149, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784660097313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11905448138713837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660111788, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1194983646273613, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660126386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12191729247570038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660140880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365217924118042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660155287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11272569000720978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660169684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11681342124938965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660184126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955440044403076, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660198513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11813142150640488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660212681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10781590640544891, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660226799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11669033020734787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660240962, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11498109996318817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660254917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11550657451152802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660268946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11147318035364151, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660283392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254209280014038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660283392, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660283393, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660366271, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.729306697845459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660366272, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660366272, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784660380797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12321529537439346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660395377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1152920350432396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660409669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11614107340574265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660424458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12055301666259766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660438953, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885833948850632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660453319, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100258082151413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660467379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.111620232462883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660482424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10790517926216125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660497569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11062797904014587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660513219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547921597957611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660528434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12294028699398041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660543492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482632905244827, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660558831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12679077684879303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660574182, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135312914848328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660574183, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660574183, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660656073, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.749374270439148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660656073, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660656074, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784660671382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12592677772045135, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660686470, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11298660188913345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660701445, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11713558435440063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660716816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1122908815741539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660731949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11885841935873032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660747425, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11802258342504501, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660763118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12198573350906372, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660778608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11951153725385666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660793992, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11682331562042236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660809646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11346852779388428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660824834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11578354984521866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660839796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607906222343445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660854665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1253659874200821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660869748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11433738470077515, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660869749, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660869749, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660952189, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7550464868545532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660952190, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660952190, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784660966975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12052910029888153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660981924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11988215148448944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784660996784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11354975402355194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661011890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11399007588624954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661026666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768643021583557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661041501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408544331789017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661056313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11708945035934448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661070825, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100886762142181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661085739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1092962920665741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661100433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10756061971187592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661115519, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12273620069026947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661130637, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577965527772903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661145648, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12197785824537277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661160940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12049926817417145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661160941, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661160941, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661244925, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7584042549133301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661244927, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661244927, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784661259928, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11893458664417267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661274700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11993011087179184, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661289450, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1145906075835228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661305153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11546212434768677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661320243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10264112055301666, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661335294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10448382794857025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661350243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11926914751529694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661365043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11485368758440018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661379848, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10331116616725922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661394578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10312729328870773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661409150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695435851812363, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661423893, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214709281921387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661438616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11181969940662384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661453328, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11482857167720795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661453382, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661453382, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661538413, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7609966993331909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661538414, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661538414, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784661553119, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475766658782959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661568142, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10692141950130463, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661583300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11610450595617294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661598090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11034782230854034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661613078, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072847992181778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661628363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11457620561122894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74383360, "lr": 3.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661643268, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11302147805690765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74547200, "lr": 3.7912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661657872, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192953586578369, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74711040, "lr": 3.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661672839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11615899205207825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74874880, "lr": 3.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661687383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1111089289188385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75038720, "lr": 3.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661702015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219135135412216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75202560, "lr": 3.8245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661716660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11339563131332397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75366400, "lr": 3.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661731169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074664369225502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75530240, "lr": 3.8412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661745855, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706400126218796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75694080, "lr": 3.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661745856, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661745856, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661829611, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7616677284240723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661829612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661829612, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 75694080, "epoch_num": 0.033042129190089795}} +:::MLLOG {"namespace": "", "time_ms": 1784661844178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10644214600324631, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 75857920, "lr": 3.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661858933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413531005382538, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76021760, "lr": 3.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661873659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09591548144817352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76185600, "lr": 3.8745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661888234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037893146276474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76349440, "lr": 3.8829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661902730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11315039545297623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76513280, "lr": 3.8912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661917115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10508042573928833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76677120, "lr": 3.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661931784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08885429799556732, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 76840960, "lr": 3.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661946255, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10938652604818344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77004800, "lr": 3.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661960837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209403932094574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77168640, "lr": 3.9245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661975543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1162818968296051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77332480, "lr": 3.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784661990446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10724795609712601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77496320, "lr": 3.9412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662004973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10102424025535583, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77660160, "lr": 3.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662020036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10145585983991623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77824000, "lr": 3.9579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662034968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10765603929758072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 77987840, "lr": 3.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662034969, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662034969, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662117821, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627230286598206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662117822, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662117822, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 77987840, "epoch_num": 0.03404340583221373}} +:::MLLOG {"namespace": "", "time_ms": 1784662132424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11666707694530487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78151680, "lr": 3.9745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662147085, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11216597259044647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78315520, "lr": 3.9829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662161063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10321582108736038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78479360, "lr": 3.9912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662176181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10610475391149521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78643200, "lr": 3.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662190622, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09960445016622543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78807040, "lr": 4.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662205390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347115993499756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 78970880, "lr": 4.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662219818, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10669748485088348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79134720, "lr": 4.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662234500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083209440112114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79298560, "lr": 4.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662249009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09805843234062195, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79462400, "lr": 4.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662263676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10889076441526413, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79626240, "lr": 4.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662278046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622848570346832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79790080, "lr": 4.0579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662292467, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10739195346832275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 79953920, "lr": 4.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662306890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10868535190820694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80117760, "lr": 4.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662321338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11396551877260208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80281600, "lr": 4.0829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662321339, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662321339, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662404423, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7608395218849182, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662404424, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662404424, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 80281600, "epoch_num": 0.03504468247433766}} +:::MLLOG {"namespace": "", "time_ms": 1784662419015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11174838989973068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80445440, "lr": 4.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662433330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973549842834473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80609280, "lr": 4.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662447853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277803987264633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80773120, "lr": 4.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662462389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10618622601032257, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 80936960, "lr": 4.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662477322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11110015958547592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81100800, "lr": 4.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662492092, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402767151594162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81264640, "lr": 4.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662506651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09963998198509216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81428480, "lr": 4.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662521405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10695674270391464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81592320, "lr": 4.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662536329, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10775955766439438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81756160, "lr": 4.1579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662550696, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078806146979332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 81920000, "lr": 4.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662565359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239707469940186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82083840, "lr": 4.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662579472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09402789920568466, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82247680, "lr": 4.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662594599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.098151795566082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82411520, "lr": 4.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662609004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761359333992004, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82575360, "lr": 4.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662609004, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662609005, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662692292, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7633724212646484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662692293, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662692293, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 82575360, "epoch_num": 0.03604595911646159}} +:::MLLOG {"namespace": "", "time_ms": 1784662706667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09221290796995163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82739200, "lr": 4.2079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662721418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10456705093383789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 82903040, "lr": 4.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662735916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11057783663272858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83066880, "lr": 4.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662750767, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0968879833817482, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83230720, "lr": 4.2329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662765195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097344011068344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83394560, "lr": 4.2412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662779873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11074347048997879, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83558400, "lr": 4.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662794515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09740129113197327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83722240, "lr": 4.2579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662809017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11151837557554245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 83886080, "lr": 4.2662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662823659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069561019539833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84049920, "lr": 4.2745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662838270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10577663034200668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84213760, "lr": 4.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662853128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1090664267539978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84377600, "lr": 4.2912499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662867412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1051446869969368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84541440, "lr": 4.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662881720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09785065054893494, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84705280, "lr": 4.3079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662896363, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728604346513748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 84869120, "lr": 4.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784662896364, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662896364, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662979777, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7627716064453125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662979778, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662979778, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 84869120, "epoch_num": 0.037047235758585526}} +:::MLLOG {"namespace": "", "time_ms": 1784662994597, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440170019865036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85032960, "lr": 4.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663009553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572925955057144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85196800, "lr": 4.3329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663024034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1058088019490242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85360640, "lr": 4.3412499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663038827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10500079393386841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85524480, "lr": 4.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663053775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09892479330301285, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85688320, "lr": 4.3579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663068317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10069633275270462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 85852160, "lr": 4.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663083522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657774657011032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86016000, "lr": 4.3745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663097822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09618804603815079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86179840, "lr": 4.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663112765, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10838782042264938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86343680, "lr": 4.3912499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663127657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10851167142391205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86507520, "lr": 4.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663142680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11627639085054398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86671360, "lr": 4.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663157763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11476539820432663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86835200, "lr": 4.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663172636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11649604141712189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 86999040, "lr": 4.4245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663187501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11593416333198547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87162880, "lr": 4.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663187502, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663187502, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663271311, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7649332284927368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663271312, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663271312, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 87162880, "epoch_num": 0.03804851240070946}} +:::MLLOG {"namespace": "", "time_ms": 1784663285929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10165029019117355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87326720, "lr": 4.4412499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663300531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11584054678678513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87490560, "lr": 4.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663314858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10806572437286377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87654400, "lr": 4.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663329451, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10186579078435898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87818240, "lr": 4.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663343984, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.091153085231781, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 87982080, "lr": 4.4745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663358682, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09901586174964905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88145920, "lr": 4.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663373202, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478005558252335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88309760, "lr": 4.4912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663387920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737283527851105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88473600, "lr": 4.4995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663402392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701005160808563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88637440, "lr": 4.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663416943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10064917057752609, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88801280, "lr": 4.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663432061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11469180881977081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 88965120, "lr": 4.5245833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663447072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10082987695932388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89128960, "lr": 4.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663462044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1101938784122467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89292800, "lr": 4.5412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663476647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212252080440521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89456640, "lr": 4.5495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663476647, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663476648, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663559025, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7635629773139954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663559026, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663559026, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 89456640, "epoch_num": 0.03904978904283339}} +:::MLLOG {"namespace": "", "time_ms": 1784663573713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842915624380112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89620480, "lr": 4.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663588433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023298054933548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89784320, "lr": 4.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663603044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11100520938634872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 89948160, "lr": 4.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663618131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09234896302223206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90112000, "lr": 4.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663632700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148853600025177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90275840, "lr": 4.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663647191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696152597665787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90439680, "lr": 4.5995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663662407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239062994718552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90603520, "lr": 4.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663677588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207226663827896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90767360, "lr": 4.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663693192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12206222116947174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 90931200, "lr": 4.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663708620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1055164709687233, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91095040, "lr": 4.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663724024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128450408577919, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91258880, "lr": 4.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663739557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12319038063287735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91422720, "lr": 4.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663754967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0995035395026207, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91586560, "lr": 4.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663770239, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11088892817497253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91750400, "lr": 4.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663770240, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663770240, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663851712, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7636770606040955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663851713, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663851713, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 91750400, "epoch_num": 0.040051065684957324}} +:::MLLOG {"namespace": "", "time_ms": 1784663866775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10414852946996689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 91914240, "lr": 4.6745833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663882109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11911071836948395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92078080, "lr": 4.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663897316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10918042808771133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92241920, "lr": 4.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663912828, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1083282083272934, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92405760, "lr": 4.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663928200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09692811965942383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92569600, "lr": 4.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663943652, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11200051009654999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92733440, "lr": 4.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663959336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10653311759233475, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 92897280, "lr": 4.7245833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663974986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10634110867977142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93061120, "lr": 4.7329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784663990274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11715427041053772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93224960, "lr": 4.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664005270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10033764690160751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93388800, "lr": 4.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664020727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1254924088716507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93552640, "lr": 4.7579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664035862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875643044710159, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93716480, "lr": 4.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664050662, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11829700320959091, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 93880320, "lr": 4.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664065421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10808556526899338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94044160, "lr": 4.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664065422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664065422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664147277, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7634211182594299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664147278, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664147278, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 94044160, "epoch_num": 0.041052342327081257}} +:::MLLOG {"namespace": "", "time_ms": 1784664162349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11417596787214279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94208000, "lr": 4.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664177453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11557183414697647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94371840, "lr": 4.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664192422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184106767177582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94535680, "lr": 4.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664207578, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11848663538694382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94699520, "lr": 4.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664222531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11471235752105713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 94863360, "lr": 4.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664237579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11965975165367126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95027200, "lr": 4.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664252749, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12721100449562073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95191040, "lr": 4.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664267439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1211310550570488, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95354880, "lr": 4.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664282167, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11690991371870041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95518720, "lr": 4.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664297153, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044881492853165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95682560, "lr": 4.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664311853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10819205641746521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 95846400, "lr": 4.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664326964, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10859448462724686, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96010240, "lr": 4.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664342018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11575690656900406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96174080, "lr": 4.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664357196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11465172469615936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96337920, "lr": 4.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664357197, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664357197, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664437991, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7659510374069214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664437992, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664437992, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 96337920, "epoch_num": 0.04205361896920519}} +:::MLLOG {"namespace": "", "time_ms": 1784664453183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11958649009466171, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96501760, "lr": 4.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664468302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10836153477430344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96665600, "lr": 4.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664483192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10899878293275833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96829440, "lr": 4.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664498560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842183232307434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 96993280, "lr": 4.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664513795, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11150064319372177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97157120, "lr": 4.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664528946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11017884314060211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97320960, "lr": 4.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664544151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11389236152172089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97484800, "lr": 4.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664559309, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10761227458715439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97648640, "lr": 4.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664574057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10687749832868576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97812480, "lr": 4.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664588469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1006854921579361, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 97976320, "lr": 4.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664603414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10476640611886978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98140160, "lr": 4.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664618000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09717684239149094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98304000, "lr": 4.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664632611, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501432418823242, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98467840, "lr": 5.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664647105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10089010000228882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98631680, "lr": 5.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664647106, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664647107, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664727540, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7678549885749817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664727541, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664727541, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 98631680, "epoch_num": 0.04305489561132912}} +:::MLLOG {"namespace": "", "time_ms": 1784664742041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10962226986885071, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98795520, "lr": 5.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664756689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10251064598560333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 98959360, "lr": 5.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664771015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09690315276384354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99123200, "lr": 5.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664785547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866735130548477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99287040, "lr": 5.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664800271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151350885629654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99450880, "lr": 5.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664814789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048690602183342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99614720, "lr": 5.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664829236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10547244548797607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99778560, "lr": 5.074583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664843836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10513937473297119, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 99942400, "lr": 5.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664858350, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219498723745346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100106240, "lr": 5.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664873059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12082646787166595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100270080, "lr": 5.099583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664887808, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11711021512746811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100433920, "lr": 5.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664902458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148157924413681, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100597760, "lr": 5.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664917338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121981799602509, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100761600, "lr": 5.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664931709, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12046225368976593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 100925440, "lr": 5.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784664931710, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784664931710, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665012784, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7680723071098328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665012785, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665012785, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 100925440, "epoch_num": 0.04405617225345306}} +:::MLLOG {"namespace": "", "time_ms": 1784665027811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11144648492336273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101089280, "lr": 5.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665042494, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09666293859481812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101253120, "lr": 5.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665057265, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10501262545585632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101416960, "lr": 5.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665072023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10360636562108994, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101580800, "lr": 5.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665086466, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11103497445583344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101744640, "lr": 5.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665101381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10557538270950317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 101908480, "lr": 5.182916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665116324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10661054402589798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102072320, "lr": 5.191249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665131190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11239687353372574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102236160, "lr": 5.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665146305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10485448688268661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102400000, "lr": 5.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665161233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10847975313663483, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102563840, "lr": 5.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665176036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10611219704151154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102727680, "lr": 5.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665190747, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1151164323091507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 102891520, "lr": 5.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665205229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10677815973758698, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103055360, "lr": 5.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665219812, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10274917632341385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103219200, "lr": 5.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665219813, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665219813, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665301367, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.768868625164032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665301368, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665301368, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 103219200, "epoch_num": 0.045057448895576994}} +:::MLLOG {"namespace": "", "time_ms": 1784665316023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005469560623169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103383040, "lr": 5.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665330889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09514143317937851, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103546880, "lr": 5.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665345503, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09628257900476456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103710720, "lr": 5.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665360277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0973416268825531, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 103874560, "lr": 5.282916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665374843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09738261997699738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104038400, "lr": 5.291249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665389849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0938575267791748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104202240, "lr": 5.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665404454, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10039786994457245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104366080, "lr": 5.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665419100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09716975688934326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104529920, "lr": 5.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665434001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10314284265041351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104693760, "lr": 5.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665448839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948928445577621, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 104857600, "lr": 5.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665463993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049174889922142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105021440, "lr": 5.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665478745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10401095449924469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105185280, "lr": 5.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665493628, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403292417526245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105349120, "lr": 5.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665508851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228300631046295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105512960, "lr": 5.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665508852, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665508853, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665590716, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696759700775146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665590717, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665590717, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 105512960, "epoch_num": 0.04605872553770093}} +:::MLLOG {"namespace": "", "time_ms": 1784665605649, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09893445670604706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105676800, "lr": 5.374583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665620448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815548151731491, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 105840640, "lr": 5.382916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665635604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002690345048904, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106004480, "lr": 5.391249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665650301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190070420503616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106168320, "lr": 5.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665665166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1011393815279007, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106332160, "lr": 5.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665680100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10807599127292633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106496000, "lr": 5.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665694863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114687472581863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106659840, "lr": 5.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665709823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10691943019628525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106823680, "lr": 5.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665724847, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10288527607917786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 106987520, "lr": 5.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665739274, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162779688835144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107151360, "lr": 5.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665753680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1003788635134697, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107315200, "lr": 5.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665768209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10305988043546677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107479040, "lr": 5.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665782813, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081109941005707, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107642880, "lr": 5.474583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665797327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11426202952861786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107806720, "lr": 5.482916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665797327, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665797328, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665879711, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7689056396484375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665879711, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665879712, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 107806720, "epoch_num": 0.04706000217982486}} +:::MLLOG {"namespace": "", "time_ms": 1784665894377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11046869307756424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 107970560, "lr": 5.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665908884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10699737071990967, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108134400, "lr": 5.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665923440, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09228158742189407, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108298240, "lr": 5.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665937946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10344073176383972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108462080, "lr": 5.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665952658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10649328678846359, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108625920, "lr": 5.524583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665967166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823854804039001, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108789760, "lr": 5.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665981695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11077644675970078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 108953600, "lr": 5.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784665996276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10383147746324539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109117440, "lr": 5.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666011228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022103041410446, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109281280, "lr": 5.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666026051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10400830209255219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109445120, "lr": 5.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666041009, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10341557115316391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109608960, "lr": 5.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666055773, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895240306854248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109772800, "lr": 5.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666070974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10655924677848816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 109936640, "lr": 5.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666086143, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10932396352291107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110100480, "lr": 5.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666086144, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666086145, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666168834, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7702418565750122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666168834, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666168835, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 110100480, "epoch_num": 0.04806127882194879}} +:::MLLOG {"namespace": "", "time_ms": 1784666183897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10282351821660995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110264320, "lr": 5.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666198736, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10402597486972809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110428160, "lr": 5.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666213300, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10445063561201096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110592000, "lr": 5.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666227586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10308480262756348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110755840, "lr": 5.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666242920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10731486231088638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 110919680, "lr": 5.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666257644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10131070762872696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111083520, "lr": 5.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666272380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09898210316896439, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111247360, "lr": 5.657916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666287525, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211844742298126, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111411200, "lr": 5.666250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666302424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11229649931192398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111575040, "lr": 5.674583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666317212, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1049850583076477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111738880, "lr": 5.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666331695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10358358919620514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 111902720, "lr": 5.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666346243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10234571248292923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112066560, "lr": 5.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666361166, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1047801822423935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112230400, "lr": 5.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666375803, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757316648960114, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112394240, "lr": 5.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666375803, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666375804, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666459123, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7700169086456299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666459124, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666459125, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 112394240, "epoch_num": 0.049062555464072725}} +:::MLLOG {"namespace": "", "time_ms": 1784666474001, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10842284560203552, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112558080, "lr": 5.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666488899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10183333605527878, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112721920, "lr": 5.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666503574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10413434356451035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 112885760, "lr": 5.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666518377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152316093444824, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113049600, "lr": 5.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666533280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10563187301158905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113213440, "lr": 5.757916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666547838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11234840750694275, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113377280, "lr": 5.766250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666562737, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137961804866791, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113541120, "lr": 5.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666577513, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11044442653656006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113704960, "lr": 5.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666592608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11361492425203323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 113868800, "lr": 5.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666607842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651081800460815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114032640, "lr": 5.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666622809, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347880423069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114196480, "lr": 5.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666637653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10480798035860062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114360320, "lr": 5.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666652730, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1053089126944542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114524160, "lr": 5.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666667722, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09939322620630264, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114688000, "lr": 5.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666667723, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666667723, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666750761, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7707329392433167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666750762, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666750762, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 114688000, "epoch_num": 0.05006383210619666}} +:::MLLOG {"namespace": "", "time_ms": 1784666765509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240296274423599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 114851840, "lr": 5.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666780211, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09992178529500961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115015680, "lr": 5.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666794921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10694962739944458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115179520, "lr": 5.857916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666809227, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10115089267492294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115343360, "lr": 5.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666824574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10841885954141617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115507200, "lr": 5.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666839257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11003818362951279, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115671040, "lr": 5.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666854158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10708703100681305, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115834880, "lr": 5.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666869337, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11601662635803223, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 115998720, "lr": 5.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666884249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11402375251054764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116162560, "lr": 5.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666899097, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093954205513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116326400, "lr": 5.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666913810, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10443145036697388, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116490240, "lr": 5.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666928516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10371007770299911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116654080, "lr": 5.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666943220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1052113026380539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116817920, "lr": 5.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666957726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829927772283554, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 116981760, "lr": 5.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784666957726, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784666957727, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667041553, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.770381510257721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667041553, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667041554, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 116981760, "epoch_num": 0.05106510874832059}} +:::MLLOG {"namespace": "", "time_ms": 1784667056302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10228724032640457, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117145600, "lr": 5.957916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667071093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10028024017810822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117309440, "lr": 5.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667086023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10912340134382248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117473280, "lr": 5.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667100668, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10530008375644684, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117637120, "lr": 5.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667115154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09444864094257355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117800960, "lr": 5.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667130408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11311542242765427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 117964800, "lr": 5.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667145130, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10190565139055252, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118128640, "lr": 6.007916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667159836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10757552087306976, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118292480, "lr": 6.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667174884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355568677186966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118456320, "lr": 6.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667190082, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10386643558740616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118620160, "lr": 6.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667204915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10963695496320724, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118784000, "lr": 6.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667220041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11481548100709915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 118947840, "lr": 6.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667235070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10711006075143814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119111680, "lr": 6.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667250231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348254442214966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119275520, "lr": 6.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667250232, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667250232, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667331688, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7708614468574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667331689, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667331689, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 119275520, "epoch_num": 0.052066385390444524}} +:::MLLOG {"namespace": "", "time_ms": 1784667347215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11297699064016342, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119439360, "lr": 6.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667362322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944531857967377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119603200, "lr": 6.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667377277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018601804971695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119767040, "lr": 6.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667392254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1071934774518013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 119930880, "lr": 6.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667406845, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10158787667751312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120094720, "lr": 6.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667421388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10794094949960709, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120258560, "lr": 6.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667436950, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11872021853923798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120422400, "lr": 6.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667451943, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10895240306854248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120586240, "lr": 6.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667466972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11659026890993118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120750080, "lr": 6.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667482399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1230560690164566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 120913920, "lr": 6.149583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667497510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10768458247184753, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121077760, "lr": 6.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667512487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11111745983362198, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121241600, "lr": 6.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667527871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156846284866333, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121405440, "lr": 6.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667542959, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071041226387024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121569280, "lr": 6.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667542960, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667542961, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667625347, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7696937918663025, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667625348, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667625348, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 121569280, "epoch_num": 0.053067662032568456}} +:::MLLOG {"namespace": "", "time_ms": 1784667640714, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11376965045928955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121733120, "lr": 6.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667656081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10974347591400146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 121896960, "lr": 6.199583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667671275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11160551011562347, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122060800, "lr": 6.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667686529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10673508793115616, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122224640, "lr": 6.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667701680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439173132181168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122388480, "lr": 6.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667717044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1144833192229271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122552320, "lr": 6.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667732220, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10747753828763962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122716160, "lr": 6.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667747401, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1150098517537117, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 122880000, "lr": 6.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667763042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11219964176416397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123043840, "lr": 6.257916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667778510, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12021785974502563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123207680, "lr": 6.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667793915, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593871027231216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123371520, "lr": 6.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667808977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11749924719333649, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123535360, "lr": 6.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667824221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760162025690079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123699200, "lr": 6.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667839732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11970734596252441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 123863040, "lr": 6.299583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667839733, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667839734, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667922319, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7705992460250854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667922320, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667922320, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 123863040, "epoch_num": 0.05406893867469239}} +:::MLLOG {"namespace": "", "time_ms": 1784667937625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12023020535707474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124026880, "lr": 6.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667952729, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10861573368310928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124190720, "lr": 6.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667967753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1200103834271431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124354560, "lr": 6.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667982485, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11380978673696518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124518400, "lr": 6.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784667997373, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10809537768363953, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124682240, "lr": 6.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668012719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11547377705574036, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 124846080, "lr": 6.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668027447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11263449490070343, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125009920, "lr": 6.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668042560, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11751175671815872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125173760, "lr": 6.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668057568, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1141408234834671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125337600, "lr": 6.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668072352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643111914396286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125501440, "lr": 6.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668087259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340641230344772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125665280, "lr": 6.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668102322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10542149841785431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125829120, "lr": 6.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668116986, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699880659580231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 125992960, "lr": 6.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668131876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10573676228523254, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126156800, "lr": 6.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668131877, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668131877, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668215129, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7712219953536987, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668215130, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668215131, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 126156800, "epoch_num": 0.05507021531681632}} +:::MLLOG {"namespace": "", "time_ms": 1784668229875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11019653081893921, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126320640, "lr": 6.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668245096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12196414172649384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126484480, "lr": 6.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668259779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11053057760000229, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126648320, "lr": 6.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668274656, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11400251090526581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126812160, "lr": 6.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668289547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11607448011636734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 126976000, "lr": 6.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668304420, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958588868379593, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127139840, "lr": 6.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668319453, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088619977235794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127303680, "lr": 6.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668334408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11036823689937592, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127467520, "lr": 6.482916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668349501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11283598840236664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127631360, "lr": 6.491249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668364270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11281221359968185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127795200, "lr": 6.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668379249, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11051535606384277, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 127959040, "lr": 6.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668394404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11122410744428635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128122880, "lr": 6.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668409397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11192701756954193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128286720, "lr": 6.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668424359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10933899879455566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128450560, "lr": 6.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668424360, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668424360, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668506172, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7713875770568848, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668506173, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668506173, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 128450560, "epoch_num": 0.056071491958940255}} +:::MLLOG {"namespace": "", "time_ms": 1784668520952, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1180403009057045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128614400, "lr": 6.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668536168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11082270741462708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128778240, "lr": 6.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668551243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11096109449863434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 128942080, "lr": 6.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668566066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10347817093133926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129105920, "lr": 6.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668580941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10579667240381241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129269760, "lr": 6.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668596043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119762435555458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129433600, "lr": 6.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668610897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10837054252624512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129597440, "lr": 6.591249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668625592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862725228071213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129761280, "lr": 6.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668640448, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989568829536438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 129925120, "lr": 6.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668655216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1023208498954773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130088960, "lr": 6.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668670020, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10541413724422455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130252800, "lr": 6.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668684528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10507814586162567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130416640, "lr": 6.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668699374, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016174405813217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130580480, "lr": 6.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668714322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11673317849636078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130744320, "lr": 6.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668714323, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668714323, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668795402, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719801068305969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668795403, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668795403, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 130744320, "epoch_num": 0.05707276860106419}} +:::MLLOG {"namespace": "", "time_ms": 1784668810139, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11537590622901917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 130908160, "lr": 6.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668825036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10590831935405731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131072000, "lr": 6.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668839978, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10055369138717651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131235840, "lr": 6.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668855069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09261589497327805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131399680, "lr": 6.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668869916, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11435283720493317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131563520, "lr": 6.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668884599, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10578653216362, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131727360, "lr": 6.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668899512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10269372910261154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 131891200, "lr": 6.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668914676, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260104924440384, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132055040, "lr": 6.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668929456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10744769871234894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132218880, "lr": 6.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668944405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10385572910308838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132382720, "lr": 6.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668959301, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10999999195337296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132546560, "lr": 6.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668974276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11317694187164307, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132710400, "lr": 6.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784668989278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772264003753662, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 132874240, "lr": 6.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669004141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10772299766540527, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133038080, "lr": 6.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669004141, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669004142, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669087191, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719723582267761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669087192, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669087192, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 133038080, "epoch_num": 0.05807404524318812}} +:::MLLOG {"namespace": "", "time_ms": 1784669102023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11018897593021393, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133201920, "lr": 6.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669117164, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10327742993831635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133365760, "lr": 6.782916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669131954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11064589768648148, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133529600, "lr": 6.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669146717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1002219170331955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133693440, "lr": 6.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669162028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11641030013561249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 133857280, "lr": 6.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669176672, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11038776487112045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134021120, "lr": 6.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669191243, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10409452021121979, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134184960, "lr": 6.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669206224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10834421217441559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134348800, "lr": 6.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669221056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10862310230731964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134512640, "lr": 6.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669235357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10871060937643051, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134676480, "lr": 6.849583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669249793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11066107451915741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 134840320, "lr": 6.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669264184, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09973981231451035, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135004160, "lr": 6.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669278588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10766025632619858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135168000, "lr": 6.874583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292973, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08807706832885742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135331840, "lr": 6.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669292973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669292974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669375911, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7718803286552429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669375912, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669375912, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 135331840, "epoch_num": 0.05907532188531205}} +:::MLLOG {"namespace": "", "time_ms": 1784669390460, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09531928598880768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135495680, "lr": 6.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669405121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135758459568024, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135659520, "lr": 6.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669419727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10857824236154556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135823360, "lr": 6.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669434323, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10211155563592911, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 135987200, "lr": 6.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669449022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1026914119720459, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136151040, "lr": 6.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669463913, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09057770669460297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136314880, "lr": 6.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669478546, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1035737618803978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136478720, "lr": 6.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669492976, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09845393896102905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136642560, "lr": 6.949583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669507506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10382619500160217, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136806400, "lr": 6.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669522313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066284030675888, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 136970240, "lr": 6.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669537027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11097100377082825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137134080, "lr": 6.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669551713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829605907201767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137297920, "lr": 6.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669566480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10643890500068665, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137461760, "lr": 6.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669581093, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1037822738289833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137625600, "lr": 6.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669581093, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669581094, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669665904, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7719734311103821, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669665905, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669665905, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 137625600, "epoch_num": 0.060076598527435986}} +:::MLLOG {"namespace": "", "time_ms": 1784669680634, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0971701592206955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137789440, "lr": 7.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669695413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1056327372789383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 137953280, "lr": 7.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669710024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09511767327785492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138117120, "lr": 7.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669724691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10303428024053574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138280960, "lr": 7.032916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669739361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09866984188556671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138444800, "lr": 7.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669753717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10076683759689331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138608640, "lr": 7.049583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669767983, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10913608223199844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138772480, "lr": 7.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669783349, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151934623718262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 138936320, "lr": 7.066249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669797904, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10965941101312637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139100160, "lr": 7.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669812720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11025281250476837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139264000, "lr": 7.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669827993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10625021159648895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139427840, "lr": 7.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669843063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10498633235692978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139591680, "lr": 7.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669857946, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10944652557373047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139755520, "lr": 7.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669872833, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911870747804642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 139919360, "lr": 7.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669872834, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669872834, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669956677, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723056077957153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669956678, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669956678, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 139919360, "epoch_num": 0.06107787516955992}} +:::MLLOG {"namespace": "", "time_ms": 1784669971367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10736892372369766, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140083200, "lr": 7.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784669986129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479751229286194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140247040, "lr": 7.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670000801, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10368064045906067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140410880, "lr": 7.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670015586, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10081726312637329, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140574720, "lr": 7.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670030426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10706590116024017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140738560, "lr": 7.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670045068, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09619719535112381, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 140902400, "lr": 7.166249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670060457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10576245933771133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141066240, "lr": 7.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670075160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10419678688049316, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141230080, "lr": 7.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670089779, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1107771173119545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141393920, "lr": 7.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670104819, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09936597943305969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141557760, "lr": 7.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670119361, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09826915711164474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141721600, "lr": 7.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670134131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10279584676027298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 141885440, "lr": 7.21625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670148967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09161832183599472, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142049280, "lr": 7.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670164018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108053058385849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142213120, "lr": 7.232916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670164019, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670164019, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670248281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723597884178162, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670248282, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670248282, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 142213120, "epoch_num": 0.06207915181168386}} +:::MLLOG {"namespace": "", "time_ms": 1784670263238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001012682914734, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142376960, "lr": 7.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670278340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192383825778961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142540800, "lr": 7.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670293100, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09920132160186768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142704640, "lr": 7.257916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670308117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041120439767838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 142868480, "lr": 7.266249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670323072, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10160984843969345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143032320, "lr": 7.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670338179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275978595018387, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143196160, "lr": 7.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670352969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10300175845623016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143360000, "lr": 7.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670367901, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545822978019714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143523840, "lr": 7.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670383137, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10209396481513977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143687680, "lr": 7.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670397793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11479848623275757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 143851520, "lr": 7.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670413121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10130627453327179, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144015360, "lr": 7.324583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670427666, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10923541337251663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144179200, "lr": 7.332916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670442208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10603763163089752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144343040, "lr": 7.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670457059, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683832317590714, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144506880, "lr": 7.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670457060, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670457061, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670540742, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7723338007926941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670540743, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670540744, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 144506880, "epoch_num": 0.06308042845380779}} +:::MLLOG {"namespace": "", "time_ms": 1784670555538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1072632223367691, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144670720, "lr": 7.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670570061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09867840260267258, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144834560, "lr": 7.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670584469, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10728270560503006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 144998400, "lr": 7.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670599221, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715340077877045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145162240, "lr": 7.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670613996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10306189954280853, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145326080, "lr": 7.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670628286, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1074538379907608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145489920, "lr": 7.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670642806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09747222065925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145653760, "lr": 7.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670657577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09841570258140564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145817600, "lr": 7.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670672174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10588046908378601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 145981440, "lr": 7.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670686793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585835576057434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146145280, "lr": 7.432916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670701443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10364833474159241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146309120, "lr": 7.441250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670715925, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11059852689504623, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146472960, "lr": 7.449583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670730365, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10672973841428757, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146636800, "lr": 7.457916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670744797, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1005413755774498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146800640, "lr": 7.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670744798, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670744798, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670828544, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726564407348633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670828545, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670828545, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 146800640, "epoch_num": 0.06408170509593172}} +:::MLLOG {"namespace": "", "time_ms": 1784670843094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10412900894880295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 146964480, "lr": 7.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670857504, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10562823712825775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147128320, "lr": 7.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670871941, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09473623335361481, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147292160, "lr": 7.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670886853, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09929139167070389, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147456000, "lr": 7.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670901306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10275126248598099, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147619840, "lr": 7.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670916160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10083019733428955, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147783680, "lr": 7.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670930638, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10688253492116928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 147947520, "lr": 7.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670945571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10984687507152557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148111360, "lr": 7.532916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670960565, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1094331368803978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148275200, "lr": 7.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670975424, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019391775131226, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148439040, "lr": 7.549583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784670990264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10356694459915161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148602880, "lr": 7.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671004616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09939266741275787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148766720, "lr": 7.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671019489, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11213545501232147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 148930560, "lr": 7.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671033861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11139510571956635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149094400, "lr": 7.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671033861, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671033862, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671118621, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.772913932800293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671118622, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671118622, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 149094400, "epoch_num": 0.06508298173805566}} +:::MLLOG {"namespace": "", "time_ms": 1784671132999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10620784014463425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149258240, "lr": 7.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671148514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10502221435308456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149422080, "lr": 7.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671163473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10636048763990402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149585920, "lr": 7.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671178618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11071683466434479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149749760, "lr": 7.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671194197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10630536824464798, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 149913600, "lr": 7.624583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671209593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10865778475999832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150077440, "lr": 7.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671224426, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1085481196641922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150241280, "lr": 7.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671239195, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09819760918617249, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150405120, "lr": 7.649583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671254058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1068880558013916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150568960, "lr": 7.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671268751, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10329602658748627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150732800, "lr": 7.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671283681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10880228877067566, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 150896640, "lr": 7.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671298379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10398140549659729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151060480, "lr": 7.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671313564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10685034096240997, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151224320, "lr": 7.691249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671328621, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09633220732212067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151388160, "lr": 7.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671328671, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671328672, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671413118, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7726266980171204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671413119, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671413119, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 151388160, "epoch_num": 0.06608425838017959}} +:::MLLOG {"namespace": "", "time_ms": 1784671428115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10019858926534653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151552000, "lr": 7.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671443099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10589537769556046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151715840, "lr": 7.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671458193, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192804038524628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 151879680, "lr": 7.724583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671473377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10440424829721451, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152043520, "lr": 7.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671488407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855840146541595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152207360, "lr": 7.741249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671503250, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12445464730262756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152371200, "lr": 7.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671518228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10846596211194992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152535040, "lr": 7.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671533294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09949006885290146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152698880, "lr": 7.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671548429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1126919761300087, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 152862720, "lr": 7.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671563321, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11228539049625397, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153026560, "lr": 7.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671578535, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1148359403014183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153190400, "lr": 7.791249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671593789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09906880557537079, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153354240, "lr": 7.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671609003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10725883394479752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153518080, "lr": 7.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623929, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10529863089323044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153681920, "lr": 7.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671623973, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671623974, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671707505, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731543183326721, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671707506, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671707506, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 153681920, "epoch_num": 0.06708553502230352}} +:::MLLOG {"namespace": "", "time_ms": 1784671722573, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09904955327510834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 153845760, "lr": 7.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671737509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827695786952972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154009600, "lr": 7.832916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671752352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11316066980361938, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154173440, "lr": 7.841249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671767501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11808893084526062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154337280, "lr": 7.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671782229, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11412468552589417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154501120, "lr": 7.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671796796, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11114463210105896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154664960, "lr": 7.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671812273, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823369771242142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154828800, "lr": 7.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671827292, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12048657238483429, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 154992640, "lr": 7.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671842234, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539118200540543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155156480, "lr": 7.891249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671857631, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10855741798877716, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155320320, "lr": 7.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671872841, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11257130652666092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155484160, "lr": 7.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671887629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081510037183762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155648000, "lr": 7.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671902639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11368443816900253, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155811840, "lr": 7.924583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671917612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11170050501823425, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 155975680, "lr": 7.932916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784671917612, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784671917613, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672000936, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7730793356895447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672000937, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672000937, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 155975680, "epoch_num": 0.06808681166442745}} +:::MLLOG {"namespace": "", "time_ms": 1784672015807, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10737687349319458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156139520, "lr": 7.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672030732, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11081668734550476, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156303360, "lr": 7.949583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672045735, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447794735431671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156467200, "lr": 7.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672060775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11095447093248367, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156631040, "lr": 7.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672075860, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10894253849983215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156794880, "lr": 7.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672091132, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10854391753673553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 156958720, "lr": 7.982916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672106069, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1014312207698822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157122560, "lr": 7.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672120993, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11224207282066345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157286400, "lr": 7.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672136121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12022126466035843, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157450240, "lr": 8.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672151247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11039496958255768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157614080, "lr": 8.01625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672166230, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.108949676156044, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157777920, "lr": 8.024583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672181595, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10468072444200516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 157941760, "lr": 8.032916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672196944, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10430964082479477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158105600, "lr": 8.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672212215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10911089926958084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158269440, "lr": 8.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672212215, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672212216, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672295227, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7731490731239319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672295228, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672295228, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 158269440, "epoch_num": 0.06908808830655139}} +:::MLLOG {"namespace": "", "time_ms": 1784672310278, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11452697962522507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158433280, "lr": 8.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672325592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11933420598506927, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158597120, "lr": 8.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672340891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11448374390602112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158760960, "lr": 8.074583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672356275, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11250210553407669, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 158924800, "lr": 8.082916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672371601, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134026199579239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159088640, "lr": 8.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672386620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12185366451740265, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159252480, "lr": 8.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672402002, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11207068711519241, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159416320, "lr": 8.107916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672416826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10939701646566391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159580160, "lr": 8.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672432505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11091728508472443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159744000, "lr": 8.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672447602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09911589324474335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 159907840, "lr": 8.132916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672462790, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10985645651817322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160071680, "lr": 8.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672478375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10954012721776962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160235520, "lr": 8.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672493739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1042756661772728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160399360, "lr": 8.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672508689, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11041856557130814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160563200, "lr": 8.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672508689, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672508690, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672591700, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7734262943267822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672591701, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672591701, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 160563200, "epoch_num": 0.07008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784672606793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11355361342430115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160727040, "lr": 8.174583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672622076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918035686016083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 160890880, "lr": 8.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672637390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10849718749523163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161054720, "lr": 8.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672652541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11290755867958069, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161218560, "lr": 8.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672667620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1142139881849289, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161382400, "lr": 8.207916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672683047, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11118613183498383, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161546240, "lr": 8.216250000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672698446, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11408234387636185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161710080, "lr": 8.224583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672713987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11306217312812805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 161873920, "lr": 8.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672729038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10631493479013443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162037760, "lr": 8.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672744207, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10844055563211441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162201600, "lr": 8.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672759720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12930433452129364, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162365440, "lr": 8.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672775035, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11329955607652664, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162529280, "lr": 8.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672790719, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11293356865644455, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162693120, "lr": 8.274583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672806281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11664816737174988, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 162856960, "lr": 8.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672806281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672806282, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672888916, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7737728357315063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672888917, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672888917, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 162856960, "epoch_num": 0.07109064159079925}} +:::MLLOG {"namespace": "", "time_ms": 1784672904149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09692110121250153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163020800, "lr": 8.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672919571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11154326796531677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163184640, "lr": 8.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672934850, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12272371351718903, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163348480, "lr": 8.307916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672950262, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10642633587121964, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163512320, "lr": 8.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672965654, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11006096005439758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163676160, "lr": 8.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672980936, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10729467123746872, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840000, "lr": 8.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784672996109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1119188517332077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164003840, "lr": 8.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673011029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1124296709895134, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164167680, "lr": 8.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673026435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10328531265258789, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164331520, "lr": 8.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673041200, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11816924065351486, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164495360, "lr": 8.366249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673056080, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774576663970947, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164659200, "lr": 8.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673071276, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11155721545219421, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164823040, "lr": 8.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673086362, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09284503012895584, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 164986880, "lr": 8.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673100994, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362746566534042, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165150720, "lr": 8.399583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673100994, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673100995, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673183151, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7738905549049377, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673183152, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673183152, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 165150720, "epoch_num": 0.07209191823292319}} +:::MLLOG {"namespace": "", "time_ms": 1784673198209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10156386345624924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165314560, "lr": 8.407916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673213012, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09747668355703354, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165478400, "lr": 8.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673227694, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1156916618347168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165642240, "lr": 8.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673242296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817093402147293, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165806080, "lr": 8.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673256783, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10980986803770065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 165969920, "lr": 8.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673271787, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10421895235776901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166133760, "lr": 8.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673286619, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10800737142562866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166297600, "lr": 8.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673301572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11363232135772705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166461440, "lr": 8.466249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673316472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09965946525335312, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166625280, "lr": 8.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673331472, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09645824879407883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166789120, "lr": 8.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673346330, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09728090465068817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 166952960, "lr": 8.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673361322, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0972534716129303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167116800, "lr": 8.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673376181, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10289257764816284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167280640, "lr": 8.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673390958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10120316594839096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167444480, "lr": 8.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673390959, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673390959, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673474329, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7739414572715759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673474330, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673474330, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 167444480, "epoch_num": 0.07309319487504712}} +:::MLLOG {"namespace": "", "time_ms": 1784673489240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10378031432628632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167608320, "lr": 8.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673504238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10348521918058395, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167772160, "lr": 8.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673519053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048920676112175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 167936000, "lr": 8.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673533956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10734036564826965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168099840, "lr": 8.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673548660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10280933976173401, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168263680, "lr": 8.557916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673563830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10027893632650375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168427520, "lr": 8.566249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673579138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10976262390613556, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168591360, "lr": 8.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673593988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10550837218761444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168755200, "lr": 8.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673608837, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10705248266458511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 168919040, "lr": 8.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673623552, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10619276762008667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169082880, "lr": 8.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673638514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1046048253774643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169246720, "lr": 8.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673653160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929902642965317, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169410560, "lr": 8.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673667626, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10622316598892212, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169574400, "lr": 8.624583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673682940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11324823647737503, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169738240, "lr": 8.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673682990, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673682990, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673768183, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7740643620491028, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673768184, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673768185, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 169738240, "epoch_num": 0.07409447151717105}} +:::MLLOG {"namespace": "", "time_ms": 1784673782842, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10875066369771957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 169902080, "lr": 8.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673797770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11202535033226013, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170065920, "lr": 8.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673813114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10777626186609268, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170229760, "lr": 8.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673828392, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10745527595281601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170393600, "lr": 8.666249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673843312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10084418952465057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170557440, "lr": 8.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673858617, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10247455537319183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170721280, "lr": 8.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673873940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09626781195402145, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 170885120, "lr": 8.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673888775, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10867737978696823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171048960, "lr": 8.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673903593, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995903819799423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171212800, "lr": 8.707916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673918486, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10315845906734467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171376640, "lr": 8.716249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673933675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10152362287044525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171540480, "lr": 8.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673948464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10338819772005081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171704320, "lr": 8.732916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673963418, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0997919961810112, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 171868160, "lr": 8.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673978501, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10473611205816269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172032000, "lr": 8.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784673978548, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784673978549, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674063012, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774145781993866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674063013, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674063013, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 172032000, "epoch_num": 0.07509574815929498}} +:::MLLOG {"namespace": "", "time_ms": 1784674078076, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09750200808048248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172195840, "lr": 8.757916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674093070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10760372877120972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172359680, "lr": 8.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674107884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09966576099395752, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172523520, "lr": 8.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674122458, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10415935516357422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172687360, "lr": 8.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674137028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09436728805303574, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 172851200, "lr": 8.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674151960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1069345474243164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173015040, "lr": 8.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674167127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10365568101406097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173178880, "lr": 8.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674182225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453972220420837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173342720, "lr": 8.816249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674197688, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10515622049570084, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173506560, "lr": 8.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674212744, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11206692457199097, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173670400, "lr": 8.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674228041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691062569618225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173834240, "lr": 8.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674243608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1121593564748764, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 173998080, "lr": 8.849583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674259004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11210335791110992, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174161920, "lr": 8.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674274219, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10225799679756165, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174325760, "lr": 8.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674274274, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674274274, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674358219, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7743735909461975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674358220, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674358221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 174325760, "epoch_num": 0.07609702480141892}} +:::MLLOG {"namespace": "", "time_ms": 1784674373144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09842416644096375, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174489600, "lr": 8.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674388598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10717843472957611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174653440, "lr": 8.882916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674404044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10192105174064636, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174817280, "lr": 8.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674418677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10723217576742172, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 174981120, "lr": 8.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674433948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10092577338218689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175144960, "lr": 8.907916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674448630, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10904238373041153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175308800, "lr": 8.916249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674463476, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10068633407354355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175472640, "lr": 8.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674478551, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09929390251636505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175636480, "lr": 8.932916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674493706, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10929296165704727, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175800320, "lr": 8.941249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674508988, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10986706614494324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 175964160, "lr": 8.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674524028, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10038410127162933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176128000, "lr": 8.957916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674539106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09519659727811813, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176291840, "lr": 8.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674554067, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.096051424741745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176455680, "lr": 8.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674569128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10696622729301453, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176619520, "lr": 8.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674569129, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674569130, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674651495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7746478915214539, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674651496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674651496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 176619520, "epoch_num": 0.07709830144354285}} +:::MLLOG {"namespace": "", "time_ms": 1784674666629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10346151143312454, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176783360, "lr": 8.991249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674681716, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11374250799417496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 176947200, "lr": 8.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674696680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10105893015861511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177111040, "lr": 9.007916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674711540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592224448919296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177274880, "lr": 9.016249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674726415, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175098270177841, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177438720, "lr": 9.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674741245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10104016214609146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177602560, "lr": 9.032916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674756197, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029704660177231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177766400, "lr": 9.041249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674771010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159376263618469, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 177930240, "lr": 9.049583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674786023, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10815075039863586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178094080, "lr": 9.057916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674801158, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10193000733852386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178257920, "lr": 9.06625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674816039, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09830248355865479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178421760, "lr": 9.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674831410, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11199081689119339, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178585600, "lr": 9.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674846661, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11971462517976761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178749440, "lr": 9.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674861921, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11004943400621414, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 178913280, "lr": 9.099583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674861922, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674861922, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674945460, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.774811863899231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674945461, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674945461, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 178913280, "epoch_num": 0.07809957808566678}} +:::MLLOG {"namespace": "", "time_ms": 1784674960761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11644724756479263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179077120, "lr": 9.107916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674975547, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1015472486615181, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179240960, "lr": 9.116249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784674990960, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10003282874822617, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179404800, "lr": 9.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675006183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10101208835840225, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179568640, "lr": 9.132916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675021333, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10628940165042877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179732480, "lr": 9.141249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675036429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10004270821809769, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 179896320, "lr": 9.149583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675051295, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10095030069351196, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180060160, "lr": 9.157916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675066316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1066991314291954, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180224000, "lr": 9.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675080831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10635633766651154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180387840, "lr": 9.174583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675095293, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09461767971515656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180551680, "lr": 9.182916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675110868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10478954017162323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180715520, "lr": 9.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675125575, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10213448852300644, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 180879360, "lr": 9.199583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675140356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09415242075920105, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181043200, "lr": 9.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675155257, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10041484981775284, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181207040, "lr": 9.216249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675155258, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675155258, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675236713, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7749334573745728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675236714, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675236715, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 181207040, "epoch_num": 0.07910085472779071}} +:::MLLOG {"namespace": "", "time_ms": 1784675251777, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11117034405469894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181370880, "lr": 9.224583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675266753, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1132715567946434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181534720, "lr": 9.232916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675281703, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10093260556459427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181698560, "lr": 9.241249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675296635, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10707175731658936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 181862400, "lr": 9.249583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675311642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10339802503585815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182026240, "lr": 9.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675326324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039164736866951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182190080, "lr": 9.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675341217, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10651619732379913, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182353920, "lr": 9.274583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675355932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10829200595617294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182517760, "lr": 9.282916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675370728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09820959717035294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182681600, "lr": 9.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675385477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09777504950761795, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 182845440, "lr": 9.299583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675400254, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1118067279458046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183009280, "lr": 9.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675414684, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09864991903305054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183173120, "lr": 9.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675429557, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10220474749803543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183336960, "lr": 9.324583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675443938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10265675187110901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183500800, "lr": 9.332916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675443939, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675443939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675525281, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7752763628959656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675525282, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675525282, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 183500800, "epoch_num": 0.08010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784675540029, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1025301143527031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183664640, "lr": 9.341249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675554543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10853541642427444, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183828480, "lr": 9.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675569148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11175630986690521, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 183992320, "lr": 9.357916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675584191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09420539438724518, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184156160, "lr": 9.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675599056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1044461578130722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184320000, "lr": 9.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675613789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1078110933303833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184483840, "lr": 9.382916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675628579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11676914244890213, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184647680, "lr": 9.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675643291, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10121992975473404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184811520, "lr": 9.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675658077, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10580836236476898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 184975360, "lr": 9.407916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675673199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11069459468126297, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185139200, "lr": 9.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675688109, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09827464818954468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185303040, "lr": 9.424583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675702917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10823645442724228, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185466880, "lr": 9.432916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675717891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09874998033046722, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185630720, "lr": 9.441249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675732569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10217124968767166, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185794560, "lr": 9.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675732569, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675732570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675815063, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7754824161529541, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675815064, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675815064, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 185794560, "epoch_num": 0.08110340801203858}} +:::MLLOG {"namespace": "", "time_ms": 1784675830258, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11002310365438461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 185958400, "lr": 9.457916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675844939, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10475002229213715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186122240, "lr": 9.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675859387, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09229744225740433, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186286080, "lr": 9.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675874891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10066035389900208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186449920, "lr": 9.482916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675889681, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897195339202881, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186613760, "lr": 9.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675904540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11109595000743866, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186777600, "lr": 9.499583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675919824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09588682651519775, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 186941440, "lr": 9.507916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675935144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10536188632249832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187105280, "lr": 9.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675950311, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10676463693380356, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187269120, "lr": 9.524583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675965287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10001964122056961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187432960, "lr": 9.532916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675980201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10471190512180328, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187596800, "lr": 9.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784675995121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10504033416509628, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187760640, "lr": 9.549583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676009886, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11105620861053467, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 187924480, "lr": 9.557916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676024863, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11544860154390335, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188088320, "lr": 9.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676024864, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676024865, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676106938, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7756990194320679, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676106939, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676106940, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 188088320, "epoch_num": 0.08210468465416251}} +:::MLLOG {"namespace": "", "time_ms": 1784676121934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11016342043876648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188252160, "lr": 9.574583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676137126, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10517815500497818, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188416000, "lr": 9.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676152015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10453499108552933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188579840, "lr": 9.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676167115, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10884460061788559, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188743680, "lr": 9.599583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676181967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266580432653427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 188907520, "lr": 9.607916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676197168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09795045852661133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189071360, "lr": 9.616249999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676211967, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10357174277305603, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189235200, "lr": 9.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676226699, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11268553137779236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189399040, "lr": 9.632916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676241517, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129661113023758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189562880, "lr": 9.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676256174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09593787789344788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189726720, "lr": 9.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676270958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10494940727949142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 189890560, "lr": 9.657916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676286236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11288374662399292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190054400, "lr": 9.666249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676301463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10246575623750687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190218240, "lr": 9.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676316456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11834380030632019, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190382080, "lr": 9.682916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676316457, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676316457, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676400082, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7759143114089966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676400083, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676400083, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 190382080, "epoch_num": 0.08310596129628645}} +:::MLLOG {"namespace": "", "time_ms": 1784676415225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10103302448987961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190545920, "lr": 9.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676430054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784837812185287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190709760, "lr": 9.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676444948, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10537763684988022, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 190873600, "lr": 9.707916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676460053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11265068501234055, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191037440, "lr": 9.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676475156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11852937936782837, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191201280, "lr": 9.724583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676490178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09589050710201263, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191365120, "lr": 9.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676505099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10592623800039291, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191528960, "lr": 9.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676519768, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10172299295663834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191692800, "lr": 9.749583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676534879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10240672528743744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 191856640, "lr": 9.757916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676549890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1048348918557167, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192020480, "lr": 9.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676564443, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10683999955654144, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192184320, "lr": 9.774583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676579987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1100415363907814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192348160, "lr": 9.782916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676594805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10022468119859695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192512000, "lr": 9.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676609887, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11586572974920273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192675840, "lr": 9.799583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676609888, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676609888, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676693726, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7761202454566956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676693727, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676693728, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 192675840, "epoch_num": 0.08410723793841038}} +:::MLLOG {"namespace": "", "time_ms": 1784676709114, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282331496477127, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 192839680, "lr": 9.807916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676724529, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11135397106409073, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193003520, "lr": 9.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676739713, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10701335966587067, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193167360, "lr": 9.824583333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676754900, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12146719545125961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193331200, "lr": 9.832916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676769934, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11605776101350784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193495040, "lr": 9.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676785235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10266154259443283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193658880, "lr": 9.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676800436, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12019394338130951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193822720, "lr": 9.857916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676815271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139361634850502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 193986560, "lr": 9.866249999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676830264, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10587747395038605, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194150400, "lr": 9.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676845317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10679730027914047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194314240, "lr": 9.882916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676860569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10732841491699219, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194478080, "lr": 9.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676875498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10593458265066147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194641920, "lr": 9.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676890754, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10715819150209427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194805760, "lr": 9.907916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676905491, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11526186019182205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 194969600, "lr": 9.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784676905491, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676905492, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676988028, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7766252756118774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676988029, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784676988029, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 194969600, "epoch_num": 0.08510851458053431}} +:::MLLOG {"namespace": "", "time_ms": 1784677003344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11269552260637283, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195133440, "lr": 9.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677018228, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10479175299406052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195297280, "lr": 9.932916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677033233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10255935788154602, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195461120, "lr": 9.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677048385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11658257246017456, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195624960, "lr": 9.949583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677063692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10928040742874146, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195788800, "lr": 9.957916666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677078919, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10650421679019928, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 195952640, "lr": 9.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677094487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10585217922925949, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196116480, "lr": 9.974583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677109789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11428233981132507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196280320, "lr": 9.982916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677125620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11865545064210892, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196444160, "lr": 9.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677141271, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10637788474559784, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196608000, "lr": 9.999583333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784677156880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11728523671627045, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196771840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677172393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11240235716104507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 196935680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677187543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10801810771226883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197099520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677203201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11617656797170639, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197263360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677203201, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677203202, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677284648, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7765882015228271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677284650, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677284650, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 197263360, "epoch_num": 0.08610979122265824}} +:::MLLOG {"namespace": "", "time_ms": 1784677299827, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10159323364496231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197427200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677315383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11134731024503708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197591040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677330543, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11545737832784653, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197754880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677345877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10166124999523163, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 197918720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677361604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998909384012222, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198082560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677377131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10852422565221786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198246400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677392555, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0989702045917511, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198410240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677407875, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10774719715118408, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198574080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677423285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10162489116191864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198737920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677438564, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10572223365306854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 198901760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677453824, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11468837410211563, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199065600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677469272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11083430051803589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199229440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677484520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11780449002981186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199393280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677500183, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328250914812088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199557120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677500183, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677500184, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677582495, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7768887877464294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677582496, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677582496, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 199557120, "epoch_num": 0.08711106786478218}} +:::MLLOG {"namespace": "", "time_ms": 1784677597868, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11260362714529037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199720960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677613399, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10710033029317856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 199884800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677628647, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11191307008266449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200048640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677644086, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1168864294886589, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200212480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677659509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11226239800453186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200376320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677675344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11720720678567886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200540160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677691272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09987843036651611, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200704000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677707162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10805065184831619, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 200867840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677722982, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10831151902675629, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201031680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677738748, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1096518412232399, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201195520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677754712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1190633475780487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201359360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677770629, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1128903180360794, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201523200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677786283, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10490678995847702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201687040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677802027, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1112578809261322, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 201850880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677802027, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677802028, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677886434, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7772077918052673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677886435, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677886435, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 201850880, "epoch_num": 0.08811234450690612}} +:::MLLOG {"namespace": "", "time_ms": 1784677901691, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10544899106025696, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202014720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677917346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106438934803009, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202178560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677932763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11896926164627075, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202342400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677948509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10666777193546295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202506240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677963402, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10657111555337906, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202670080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677978505, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10218729823827744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202833920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784677994340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1067143902182579, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 202997760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678009435, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1106976792216301, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203161600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678024604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11403150111436844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203325440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678039569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09465464949607849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203489280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678054879, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11718034744262695, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203653120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678070338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10139206796884537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203816960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678085380, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09830401092767715, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 203980800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678100559, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10825882107019424, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204144640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678100560, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678100560, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678186352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7771744728088379, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678186353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678186353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 204144640, "epoch_num": 0.08911362114903006}} +:::MLLOG {"namespace": "", "time_ms": 1784678201459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10277565568685532, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204308480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678216644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09307795017957687, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204472320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678231881, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09718520194292068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204636160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678246980, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10866881906986237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204800000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678262216, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1089988574385643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 204963840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678277419, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11254843324422836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205127680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678292190, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09643679112195969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205291520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678306999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10647602379322052, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205455360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678321947, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10722125321626663, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205619200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678337030, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11162400245666504, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205783040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678352054, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09887947887182236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 205946880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678367862, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09962727874517441, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206110720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678383103, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1116517186164856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206274560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678398192, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09812911599874496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206438400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678398192, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678398193, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678483130, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7773910760879517, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678483132, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678483132, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 206438400, "epoch_num": 0.09011489779115399}} +:::MLLOG {"namespace": "", "time_ms": 1784678498540, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626372694969177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206602240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678513726, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09637433290481567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206766080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678529386, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10539542138576508, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 206929920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678545304, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09630221128463745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207093760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678561016, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10296431928873062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207257600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678576602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11279483139514923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207421440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678592063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10827206820249557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207585280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678606849, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09551886469125748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207749120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678621954, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11608303338289261, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 207912960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678637131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10439218580722809, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208076800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678651899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1039082258939743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208240640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678668004, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10151820629835129, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208404480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678683159, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10283852368593216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208568320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678698452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10214132070541382, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208732160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678698453, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678698453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678781362, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7774686813354492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678781363, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678781363, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 208732160, "epoch_num": 0.09111617443327792}} +:::MLLOG {"namespace": "", "time_ms": 1784678796897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10137844830751419, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 208896000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678812308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09940645843744278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209059840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678827686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10260240733623505, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209223680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678842834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10340050607919693, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209387520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678858215, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09877457469701767, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209551360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678873407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10148677229881287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209715200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678888411, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09691827744245529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 209879040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678903281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09914998710155487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210042880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678918499, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10362300276756287, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210206720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678933612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10811429470777512, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210370560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678948680, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10432843863964081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210534400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678963653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10778222978115082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210698240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678978761, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10219649225473404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 210862080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678993675, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09671175479888916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211025920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784678993676, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784678993676, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679075320, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7775768041610718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679075321, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679075321, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 211025920, "epoch_num": 0.09211745107540185}} +:::MLLOG {"namespace": "", "time_ms": 1784679090492, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09491749107837677, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211189760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679105302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09981236606836319, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211353600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679120236, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09770157933235168, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211517440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679134890, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0996452122926712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211681280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679149840, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10483528673648834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 211845120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679165191, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09613760560750961, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212008960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679180128, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1027396023273468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212172800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679195245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09823426604270935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212336640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679210148, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0983203873038292, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212500480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679225094, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1050875335931778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212664320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679240025, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09324636310338974, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212828160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679255325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10817120969295502, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 212992000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679270572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10574782639741898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213155840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679285830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1088656634092331, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213319680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679285830, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679285831, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679367683, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7776129245758057, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679367684, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679367684, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 213319680, "epoch_num": 0.09311872771752579}} +:::MLLOG {"namespace": "", "time_ms": 1784679382653, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10510575771331787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213483520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679397592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10221753269433975, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213647360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679412677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10604342818260193, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213811200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679428099, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1031530424952507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 213975040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679442877, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09678778797388077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214138880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679457805, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10528190433979034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214302720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679472851, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.0953587219119072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214466560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679487583, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11445263773202896, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214630400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679502487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09884084761142731, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214794240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679517891, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09036648273468018, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 214958080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679533129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10046559572219849, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215121920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679548302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10545393079519272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215285760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679563287, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10449913144111633, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215449600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679578344, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10897134989500046, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215613440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679578344, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679578345, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679659412, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7777277231216431, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679659412, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679659413, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 215613440, "epoch_num": 0.09412000435964972}} +:::MLLOG {"namespace": "", "time_ms": 1784679674497, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10167291015386581, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215777280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679689327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10626128315925598, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 215941120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679704434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09784206002950668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216104960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679719935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09998448193073273, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216268800, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679735463, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10053589940071106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216432640, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679750669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09521231055259705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216596480, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679765902, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10607171803712845, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216760320, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679781305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10298068076372147, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 216924160, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679796407, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1001158282160759, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217088000, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679812034, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10958541184663773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217251840, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679827117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09809023141860962, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217415680, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679842437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10129140317440033, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217579520, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679857390, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1007080227136612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217743360, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679872395, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10551941394805908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 217907200, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679872396, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679872396, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679955352, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7779159545898438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679955353, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679955353, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 217907200, "epoch_num": 0.09512128100177365}} +:::MLLOG {"namespace": "", "time_ms": 1784679970397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257373750209808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218071040, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784679985506, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09525905549526215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218234880, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680000670, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09882068634033203, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218398720, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680015651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10839606076478958, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218562560, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680031031, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10355225950479507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218726400, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680046316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.08801201730966568, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 218890240, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680061924, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10591001063585281, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219054080, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680077541, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09257597476243973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219217920, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680092999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10417633503675461, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219381760, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680108414, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09565775841474533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219545600, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680123793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09650707989931107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219709440, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680138876, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10291439294815063, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 219873280, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680154209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11214839667081833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220037120, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680169389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.09995894134044647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 220200960, "lr": 1e-06}} +:::MLLOG {"namespace": "", "time_ms": 1784680169389, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} +:::MLLOG {"namespace": "", "time_ms": 1784680169390, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 220200960, "epoch_num": 0.09612255764389759}} diff --git a/recommendation_v4/rcp_logs/gbs_8192/seed94090156.log b/recommendation_v4/rcp_logs/gbs_8192/seed94090156.log new file mode 100644 index 000000000..ed1259629 --- /dev/null +++ b/recommendation_v4/rcp_logs/gbs_8192/seed94090156.log @@ -0,0 +1,632 @@ +:::MLLOG {"namespace": "", "time_ms": 1784940780263, "event_type": "POINT_IN_TIME", "key": "cache_clear", "value": true, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 104}} +:::MLLOG {"namespace": "", "time_ms": 1784940780263, "event_type": "INTERVAL_START", "key": "init_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/train_ranker.py", "lineno": 105}} +:::MLLOG {"namespace": "", "time_ms": 1784940801844, "event_type": "POINT_IN_TIME", "key": "submission_benchmark", "value": "hstu", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 149}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "submission_org", "value": "AMD", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 150}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "submission_division", "value": "closed", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 151}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "submission_status", "value": "onprem", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 152}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "submission_platform", "value": "MI350", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 153}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "global_batch_size", "value": 8192, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 171}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "gradient_accumulation_steps", "value": 1, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 172}} +:::MLLOG {"namespace": "", "time_ms": 1784940801845, "event_type": "POINT_IN_TIME", "key": "seed", "value": 94090156, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 175}} +:::MLLOG {"namespace": "", "time_ms": 1784940801846, "event_type": "POINT_IN_TIME", "key": "opt_name", "value": "Adam", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 176}} +:::MLLOG {"namespace": "", "time_ms": 1784940801846, "event_type": "POINT_IN_TIME", "key": "opt_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 180}} +:::MLLOG {"namespace": "", "time_ms": 1784940801846, "event_type": "POINT_IN_TIME", "key": "opt_sparse_name", "value": "RowWiseAdagrad", "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 184}} +:::MLLOG {"namespace": "", "time_ms": 1784940801846, "event_type": "POINT_IN_TIME", "key": "opt_sparse_base_learning_rate", "value": 1e-06, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 190}} +:::MLLOG {"namespace": "", "time_ms": 1784940801850, "event_type": "INTERVAL_END", "key": "init_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 196}} +:::MLLOG {"namespace": "", "time_ms": 1784940801850, "event_type": "INTERVAL_START", "key": "run_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 197}} +:::MLLOG {"namespace": "", "time_ms": 1784941725157, "event_type": "POINT_IN_TIME", "key": "train_samples", "value": 2290835423, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 265}} +:::MLLOG {"namespace": "", "time_ms": 1784941725158, "event_type": "POINT_IN_TIME", "key": "eval_samples", "value": 2058204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 267}} +:::MLLOG {"namespace": "", "time_ms": 1784941725528, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 0, "epoch_num": 0.0}} +:::MLLOG {"namespace": "", "time_ms": 1784941851421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13905055820941925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 163840, "lr": 7.916666666666666e-10}} +:::MLLOG {"namespace": "", "time_ms": 1784941864933, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886143267154694, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 327680, "lr": 1.6249999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941878903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13842469453811646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 491520, "lr": 2.458333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941893021, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13858996331691742, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 655360, "lr": 3.2916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941907049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13874505460262299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 819200, "lr": 4.125e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941921129, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13852688670158386, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 983040, "lr": 4.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941935144, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13865657150745392, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1146880, "lr": 5.7916666666666664e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941949290, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13879381120204926, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1310720, "lr": 6.625e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941963308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385953724384308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1474560, "lr": 7.458333333333332e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941977185, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13871899247169495, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1638400, "lr": 8.291666666666665e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784941991717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1385965198278427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1802240, "lr": 9.124999999999999e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942005949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13881181180477142, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 1966080, "lr": 9.958333333333333e-09}} +:::MLLOG {"namespace": "", "time_ms": 1784942020528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853195309638977, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2129920, "lr": 1.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942034923, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13886302709579468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2293760, "lr": 1.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942034923, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942034924, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942131914, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5030940771102905, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942131915, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942131915, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 2293760, "epoch_num": 0.0010012766421239332}} +:::MLLOG {"namespace": "", "time_ms": 1784942146169, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13853797316551208, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2457600, "lr": 1.2458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942160316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13855013251304626, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2621440, "lr": 1.3291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942174884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13834108412265778, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2785280, "lr": 1.4124999999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942188280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13857272267341614, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 2949120, "lr": 1.4958333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942201981, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381465345621109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3112960, "lr": 1.5791666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942216806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1382952183485031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3276800, "lr": 1.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942230500, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13827921450138092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3440640, "lr": 1.7458333333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942244412, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381874680519104, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3604480, "lr": 1.8291666666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942259083, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381726861000061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3768320, "lr": 1.9124999999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942273042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1381683647632599, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 3932160, "lr": 1.9958333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942287577, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1384214460849762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4096000, "lr": 2.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942302141, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13864989578723907, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4259840, "lr": 2.1624999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942316242, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13804666697978973, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4423680, "lr": 2.2458333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942331151, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1383395493030548, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4587520, "lr": 2.3291666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942331151, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942331152, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942426843, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5032354593276978, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942426844, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942426844, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 4587520, "epoch_num": 0.0020025532842478664}} +:::MLLOG {"namespace": "", "time_ms": 1784942441188, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13782553374767303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4751360, "lr": 2.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942455434, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13823845982551575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 4915200, "lr": 2.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942469866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137461319565773, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5079040, "lr": 2.5791666666666668e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942486725, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13684138655662537, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5242880, "lr": 2.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942501325, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13818742334842682, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5406720, "lr": 2.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942515831, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1380784660577774, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5570560, "lr": 2.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942529985, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137678325176239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5734400, "lr": 2.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942544314, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13726551830768585, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 5898240, "lr": 2.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942559062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13710835576057434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6062080, "lr": 3.0791666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942573336, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13789720833301544, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6225920, "lr": 3.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942587614, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739560544490814, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6389760, "lr": 3.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942601780, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1376650482416153, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6553600, "lr": 3.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942616830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378381848335266, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6717440, "lr": 3.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942631464, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.137699693441391, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 6881280, "lr": 3.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942631465, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942631465, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942726801, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5033757090568542, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942726802, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942726802, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 6881280, "epoch_num": 0.0030038299263717995}} +:::MLLOG {"namespace": "", "time_ms": 1784942741327, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13690634071826935, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7045120, "lr": 3.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942755658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13739846646785736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7208960, "lr": 3.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942769912, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1367815136909485, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7372800, "lr": 3.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942784692, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13733865320682526, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7536640, "lr": 3.829166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942799811, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13746240735054016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7700480, "lr": 3.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942814616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13787606358528137, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 7864320, "lr": 3.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942829459, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13661985099315643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8028160, "lr": 4.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942843987, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1374415159225464, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8192000, "lr": 4.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942858793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1369202882051468, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8355840, "lr": 4.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942873884, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13843193650245667, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8519680, "lr": 4.3291666666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942888712, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641183078289032, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8683520, "lr": 4.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942903731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13775645196437836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 8847360, "lr": 4.4958333333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942918348, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13548614084720612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9011200, "lr": 4.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942933357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13687552511692047, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9175040, "lr": 4.6624999999999996e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784942933358, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784942933358, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943026642, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5035967826843262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943026643, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943026643, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 9175040, "epoch_num": 0.004005106568495733}} +:::MLLOG {"namespace": "", "time_ms": 1784943040888, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13635388016700745, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9338880, "lr": 4.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943055154, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13555485010147095, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9502720, "lr": 4.8291666666666664e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943069620, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13825403153896332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9666560, "lr": 4.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943083883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1347111165523529, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9830400, "lr": 4.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943098385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13674448430538177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 9994240, "lr": 5.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943112834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342227041721344, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10158080, "lr": 5.1624999999999994e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943127285, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355389952659607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10321920, "lr": 5.2458333333333335e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943142233, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13544544577598572, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10485760, "lr": 5.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943157174, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378238946199417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10649600, "lr": 5.4124999999999997e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943171717, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351134479045868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10813440, "lr": 5.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943186846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14766326546669006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 10977280, "lr": 5.5791666666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943201657, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1355045735836029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11141120, "lr": 5.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943215931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404406607151031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11304960, "lr": 5.7458333333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943230421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13772538304328918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11468800, "lr": 5.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943230422, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943230422, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943326481, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5038921236991882, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943326482, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943326482, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 11468800, "epoch_num": 0.0050063832106196655}} +:::MLLOG {"namespace": "", "time_ms": 1784943340998, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354065239429474, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11632640, "lr": 5.9124999999999995e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943355272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13978120684623718, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11796480, "lr": 5.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943370156, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13596680760383606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 11960320, "lr": 6.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943384496, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13321910798549652, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12124160, "lr": 6.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943399057, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13496533036231995, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12288000, "lr": 6.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943414003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13915176689624786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12451840, "lr": 6.329166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943428096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13343356549739838, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12615680, "lr": 6.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943444224, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13287730515003204, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12779520, "lr": 6.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943459477, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1332816630601883, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 12943360, "lr": 6.579166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943473885, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13154885172843933, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13107200, "lr": 6.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943488927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13122256100177765, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13271040, "lr": 6.745833333333332e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943503683, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13511055707931519, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13434880, "lr": 6.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943518439, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13219547271728516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13598720, "lr": 6.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943533838, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13888314366340637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13762560, "lr": 6.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943533839, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943533840, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943628612, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5042330026626587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943628612, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943628613, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 13762560, "epoch_num": 0.006007659852743599}} +:::MLLOG {"namespace": "", "time_ms": 1784943643251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13360434770584106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 13926400, "lr": 7.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943657608, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340085744857788, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14090240, "lr": 7.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943672091, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1357937902212143, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14254080, "lr": 7.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943687106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13506732881069183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14417920, "lr": 7.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943701829, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1378706991672516, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14581760, "lr": 7.4125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943716462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278015524148941, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14745600, "lr": 7.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943731196, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13516132533550262, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 14909440, "lr": 7.579166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943745636, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918736040592194, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15073280, "lr": 7.6625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943760633, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350629806518555, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15237120, "lr": 7.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943776003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13253222405910492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15400960, "lr": 7.829166666666665e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943790396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13355326652526855, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15564800, "lr": 7.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943805381, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1342674195766449, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15728640, "lr": 7.995833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943820861, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353611797094345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 15892480, "lr": 8.079166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943835873, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13342984020709991, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16056320, "lr": 8.1625e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943835874, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943835874, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943930722, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5048770904541016, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943930723, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943930723, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 16056320, "epoch_num": 0.007008936494867532}} +:::MLLOG {"namespace": "", "time_ms": 1784943946062, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261187076568604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16220160, "lr": 8.245833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943961408, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13372644782066345, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16384000, "lr": 8.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943976590, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1356739103794098, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16547840, "lr": 8.412500000000001e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784943992186, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539868593215942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16711680, "lr": 8.495833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944007150, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13965333998203278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 16875520, "lr": 8.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944021727, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13318023085594177, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17039360, "lr": 8.662499999999998e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944037720, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1353289783000946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17203200, "lr": 8.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944053058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13333433866500854, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17367040, "lr": 8.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944068473, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13285872340202332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17530880, "lr": 8.912499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944083687, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13170048594474792, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17694720, "lr": 8.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944098839, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13588516414165497, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 17858560, "lr": 9.079166666666667e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944114121, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371813297271729, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18022400, "lr": 9.162499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944129046, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13335688412189484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18186240, "lr": 9.245833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944143799, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13064339756965637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18350080, "lr": 9.329166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944143799, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944143800, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944236067, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5058125853538513, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944236067, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944236068, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 18350080, "epoch_num": 0.008010213136991465}} +:::MLLOG {"namespace": "", "time_ms": 1784944251383, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13796916604042053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18513920, "lr": 9.412499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944267043, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13479822874069214, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18677760, "lr": 9.495833333333334e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944282388, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.135401651263237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 18841600, "lr": 9.579166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944297880, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13125328719615936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19005440, "lr": 9.662499999999999e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944313509, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13364379107952118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19169280, "lr": 9.745833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944328800, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13404783606529236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19333120, "lr": 9.829166666666666e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944344396, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1370343416929245, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19496960, "lr": 9.9125e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944359659, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13384605944156647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19660800, "lr": 9.995833333333333e-08}} +:::MLLOG {"namespace": "", "time_ms": 1784944374639, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329483687877655, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19824640, "lr": 1.0079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944389826, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1296517550945282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 19988480, "lr": 1.0162499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944404498, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1276158094406128, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20152320, "lr": 1.0245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944419645, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329897940158844, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20316160, "lr": 1.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944434511, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12959618866443634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20480000, "lr": 1.0412499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944449245, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12479651719331741, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20643840, "lr": 1.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944449245, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944449246, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944543872, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5068331360816956, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944543874, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944543874, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 20643840, "epoch_num": 0.009011489779115398}} +:::MLLOG {"namespace": "", "time_ms": 1784944559326, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1311798244714737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20807680, "lr": 1.0579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944574296, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13608810305595398, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 20971520, "lr": 1.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944589806, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13323654234409332, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21135360, "lr": 1.0745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944605024, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13045592606067657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21299200, "lr": 1.0829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944619360, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12830698490142822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21463040, "lr": 1.0912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944633949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12651613354682922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21626880, "lr": 1.0995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944649528, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289021521806717, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21790720, "lr": 1.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944664340, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13242484629154205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 21954560, "lr": 1.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944679357, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13542760908603668, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22118400, "lr": 1.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944694558, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1248716413974762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22282240, "lr": 1.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944708958, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285681575536728, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22446080, "lr": 1.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944723883, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13418754935264587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22609920, "lr": 1.1495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944738789, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099823892116547, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22773760, "lr": 1.1579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944754010, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12918341159820557, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 22937600, "lr": 1.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944754012, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944754012, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944848288, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5081168413162231, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944848290, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944848290, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 22937600, "epoch_num": 0.010012766421239331}} +:::MLLOG {"namespace": "", "time_ms": 1784944863700, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13040019571781158, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23101440, "lr": 1.1745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944878437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12912043929100037, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23265280, "lr": 1.1829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944892830, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13308799266815186, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23429120, "lr": 1.1912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944908413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12374423444271088, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23592960, "lr": 1.1995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944923548, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1294095516204834, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23756800, "lr": 1.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944938704, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12574510276317596, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 23920640, "lr": 1.2162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944953305, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12987980246543884, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24084480, "lr": 1.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944968377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12185632437467575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24248320, "lr": 1.2329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944983135, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13583222031593323, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24412160, "lr": 1.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784944998015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13715238869190216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24576000, "lr": 1.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945013178, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12382373958826065, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24739840, "lr": 1.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945028480, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12314941734075546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 24903680, "lr": 1.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945043382, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12547567486763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25067520, "lr": 1.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945058332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12910687923431396, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25231360, "lr": 1.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945058332, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945058333, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945152776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5097717046737671, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945152777, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945152777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 25231360, "epoch_num": 0.011014043063363265}} +:::MLLOG {"namespace": "", "time_ms": 1784945167064, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13644269108772278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25395200, "lr": 1.2912499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945182413, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1364903301000595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25559040, "lr": 1.2995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945197665, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13566599786281586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25722880, "lr": 1.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945212728, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1399025022983551, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 25886720, "lr": 1.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945227478, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13666218519210815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26050560, "lr": 1.3245833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945242733, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13814234733581543, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26214400, "lr": 1.3329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945257721, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12723416090011597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26378240, "lr": 1.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945272462, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12612593173980713, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26542080, "lr": 1.3495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945287642, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14221084117889404, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26705920, "lr": 1.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945302836, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1409851610660553, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 26869760, "lr": 1.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945318235, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12714947760105133, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27033600, "lr": 1.374583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945333352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12809398770332336, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27197440, "lr": 1.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945347955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13263250887393951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27361280, "lr": 1.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945362053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131465345621109, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27525120, "lr": 1.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945362054, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945362054, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945453486, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5118551850318909, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945453487, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945453487, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 27525120, "epoch_num": 0.012015319705487198}} +:::MLLOG {"namespace": "", "time_ms": 1784945468864, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14121124148368835, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27688960, "lr": 1.4079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945484544, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1424025595188141, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 27852800, "lr": 1.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945499693, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13532699644565582, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28016640, "lr": 1.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945514750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13957887887954712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28180480, "lr": 1.4329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945530527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14494022727012634, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28344320, "lr": 1.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945545474, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14079736173152924, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28508160, "lr": 1.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945560834, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12582120299339294, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28672000, "lr": 1.4579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945576106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13764746487140656, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28835840, "lr": 1.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945591136, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184978067874908, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 28999680, "lr": 1.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945606644, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13615870475769043, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29163520, "lr": 1.4829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945621598, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13783445954322815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29327360, "lr": 1.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945636308, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13296347856521606, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29491200, "lr": 1.4995833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945652241, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13444876670837402, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29655040, "lr": 1.5079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945667938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13627639412879944, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29818880, "lr": 1.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945667938, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945667939, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945762279, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5138501524925232, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945762280, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945762280, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 29818880, "epoch_num": 0.013016596347611131}} +:::MLLOG {"namespace": "", "time_ms": 1784945777791, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13867592811584473, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 29982720, "lr": 1.5245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945793247, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.138139009475708, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30146560, "lr": 1.5329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945808975, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13163943588733673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30310400, "lr": 1.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945824871, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13300028443336487, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30474240, "lr": 1.5495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945840168, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13378560543060303, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30638080, "lr": 1.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945856048, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14007599651813507, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30801920, "lr": 1.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945871582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13294096291065216, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 30965760, "lr": 1.5745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945887022, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14138725399971008, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31129600, "lr": 1.5829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945901866, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14082413911819458, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31293440, "lr": 1.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945916997, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1350783109664917, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31457280, "lr": 1.5995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945932199, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1278083771467209, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31621120, "lr": 1.6079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945947058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12989988923072815, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31784960, "lr": 1.61625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945961757, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13409262895584106, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 31948800, "lr": 1.6245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945977090, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12758849561214447, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32112640, "lr": 1.6329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784945977091, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784945977091, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946071584, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5161877870559692, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946071585, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946071585, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 32112640, "epoch_num": 0.014017872989735064}} +:::MLLOG {"namespace": "", "time_ms": 1784946086356, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13207630813121796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32276480, "lr": 1.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946101134, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13695663213729858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32440320, "lr": 1.6495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946116433, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347512483596802, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32604160, "lr": 1.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946131095, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13190069794654846, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32768000, "lr": 1.66625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946146225, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13383010029792786, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 32931840, "lr": 1.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946161251, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13345764577388763, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33095680, "lr": 1.6829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946176542, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13394534587860107, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33259520, "lr": 1.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946191758, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13280023634433746, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33423360, "lr": 1.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946206162, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13969773054122925, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33587200, "lr": 1.7079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946222007, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13179922103881836, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33751040, "lr": 1.71625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946237581, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12893040478229523, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 33914880, "lr": 1.724583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946252677, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13093505799770355, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34078720, "lr": 1.7329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946267822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13246136903762817, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34242560, "lr": 1.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946283316, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13608936965465546, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34406400, "lr": 1.7495833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946283317, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946283318, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946378420, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5193868279457092, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946378421, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946378421, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 34406400, "epoch_num": 0.015019149631858996}} +:::MLLOG {"namespace": "", "time_ms": 1784946393554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13348685204982758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34570240, "lr": 1.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946408569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12293566763401031, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34734080, "lr": 1.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946423955, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13050566613674164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 34897920, "lr": 1.7745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946438856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1302546113729477, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35061760, "lr": 1.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946454124, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1351708322763443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35225600, "lr": 1.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946469822, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13543429970741272, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35389440, "lr": 1.7995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946485017, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1354699581861496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35553280, "lr": 1.8079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946500865, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.123927041888237, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35717120, "lr": 1.8162500000000001e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946516032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13094298541545868, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 35880960, "lr": 1.8245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946530452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1312914490699768, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36044800, "lr": 1.8329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946545856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13025623559951782, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36208640, "lr": 1.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946561256, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13316616415977478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36372480, "lr": 1.8495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946576612, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1309370994567871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36536320, "lr": 1.8579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946591738, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13027873635292053, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36700160, "lr": 1.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946591738, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946591739, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946685496, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5237644910812378, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946685497, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946685497, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 36700160, "epoch_num": 0.01602042627398293}} +:::MLLOG {"namespace": "", "time_ms": 1784946700920, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13770505785942078, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 36864000, "lr": 1.8745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946716131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.132155179977417, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37027840, "lr": 1.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946731584, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335451304912567, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37191680, "lr": 1.8912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946746972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13046012818813324, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37355520, "lr": 1.8995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946762294, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12713100016117096, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37519360, "lr": 1.9079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946777792, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13218577206134796, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37683200, "lr": 1.9162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946793377, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13563406467437744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 37847040, "lr": 1.9245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946809088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283993303775787, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38010880, "lr": 1.9329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946824843, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13121555745601654, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38174720, "lr": 1.9412499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946840422, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13901394605636597, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38338560, "lr": 1.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946855731, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13389554619789124, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38502400, "lr": 1.9579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946870240, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1375771015882492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38666240, "lr": 1.9662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946885588, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13184355199337006, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38830080, "lr": 1.9745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946900763, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13340817391872406, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 38993920, "lr": 1.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784946900764, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946900764, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946996512, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5296505093574524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946996513, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784946996513, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 38993920, "epoch_num": 0.017021702916106864}} +:::MLLOG {"namespace": "", "time_ms": 1784947011907, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13108664751052856, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39157760, "lr": 1.9912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947026899, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13492120802402496, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39321600, "lr": 1.999583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947042332, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13779711723327637, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39485440, "lr": 2.0079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947057421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13325388729572296, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39649280, "lr": 2.0162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947072101, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12746131420135498, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39813120, "lr": 2.0245833333333331e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947087270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1292697638273239, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 39976960, "lr": 2.0329166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947102342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12593616545200348, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40140800, "lr": 2.0412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947117739, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12394209951162338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40304640, "lr": 2.0495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947133324, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13606664538383484, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40468480, "lr": 2.0579166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947148734, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1372380256652832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40632320, "lr": 2.0662499999999999e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947164214, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13472682237625122, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40796160, "lr": 2.0745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947179956, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12693139910697937, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 40960000, "lr": 2.0829166666666668e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947195272, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13070008158683777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41123840, "lr": 2.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947210061, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12359321862459183, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41287680, "lr": 2.0995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947210062, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947210062, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947302849, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5379031300544739, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947302850, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947302850, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 41287680, "epoch_num": 0.018022979558230796}} +:::MLLOG {"namespace": "", "time_ms": 1784947318650, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13243316113948822, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41451520, "lr": 2.1079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947334421, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13261711597442627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41615360, "lr": 2.11625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947349858, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13374851644039154, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41779200, "lr": 2.1245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947365312, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13802607357501984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 41943040, "lr": 2.1329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947380793, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13593114912509918, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42106880, "lr": 2.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947396105, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13134048879146576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42270720, "lr": 2.1495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947411815, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12716495990753174, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42434560, "lr": 2.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947427302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13512510061264038, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42598400, "lr": 2.16625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947442317, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247188076376915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42762240, "lr": 2.1745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947458058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033828139305115, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 42926080, "lr": 2.1829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947473393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13254795968532562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43089920, "lr": 2.1912499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947489179, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13412420451641083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43253760, "lr": 2.1995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947504897, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14033126831054688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43417600, "lr": 2.2079166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947520284, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1361795961856842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43581440, "lr": 2.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947520285, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947520285, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947614389, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5502026081085205, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947614391, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947614391, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 43581440, "epoch_num": 0.01902425620035473}} +:::MLLOG {"namespace": "", "time_ms": 1784947630404, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13641151785850525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43745280, "lr": 2.2245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947645798, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13149140775203705, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 43909120, "lr": 2.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947660530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13250961899757385, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44072960, "lr": 2.2412499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947675616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13895519077777863, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44236800, "lr": 2.2495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947691756, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12995222210884094, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44400640, "lr": 2.2579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947707572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13836424052715302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44564480, "lr": 2.2662499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947723248, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13515114784240723, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44728320, "lr": 2.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947738778, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371249556541443, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 44892160, "lr": 2.2829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947754138, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13468831777572632, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45056000, "lr": 2.2912499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947770175, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13033072650432587, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45219840, "lr": 2.2995833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947785487, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14035974442958832, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45383680, "lr": 2.3079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947801112, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13453394174575806, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45547520, "lr": 2.3162499999999998e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947816742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1289331614971161, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45711360, "lr": 2.3245833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947832189, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13330332934856415, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 45875200, "lr": 2.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947832189, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947832190, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947927494, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.564673900604248, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947927495, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947927495, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 45875200, "epoch_num": 0.020025532842478662}} +:::MLLOG {"namespace": "", "time_ms": 1784947943582, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.14185789227485657, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46039040, "lr": 2.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947959070, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13275563716888428, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46202880, "lr": 2.3495833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947975056, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13347461819648743, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46366720, "lr": 2.3579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784947990999, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13032373785972595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46530560, "lr": 2.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948006750, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1352643519639969, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46694400, "lr": 2.3745833333333332e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948022618, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1335311383008957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 46858240, "lr": 2.3829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948038816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13399918377399445, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47022080, "lr": 2.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948054587, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13539493083953857, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47185920, "lr": 2.399583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948070209, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13099004328250885, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47349760, "lr": 2.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948086277, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13264256715774536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47513600, "lr": 2.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948102032, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13283459842205048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47677440, "lr": 2.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948118044, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13698606193065643, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 47841280, "lr": 2.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948133516, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1358991265296936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48005120, "lr": 2.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948149281, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1371888816356659, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48168960, "lr": 2.4495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948149281, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948149282, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948243922, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.5798835158348083, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948243923, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948243923, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 48168960, "epoch_num": 0.021026809484602595}} +:::MLLOG {"namespace": "", "time_ms": 1784948259784, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315540224313736, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48332800, "lr": 2.4579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948275771, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13503558933734894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48496640, "lr": 2.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948291752, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13371476531028748, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48660480, "lr": 2.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948308063, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13192249834537506, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48824320, "lr": 2.4829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948323674, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13691174983978271, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 48988160, "lr": 2.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948339042, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12941157817840576, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49152000, "lr": 2.4995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948355171, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13350863754749298, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49315840, "lr": 2.507916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948370397, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12351956218481064, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49479680, "lr": 2.5162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948385972, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13092103600502014, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49643520, "lr": 2.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948401041, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.131168931722641, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49807360, "lr": 2.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948416530, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13104663789272308, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 49971200, "lr": 2.54125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948431502, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13129375874996185, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50135040, "lr": 2.5495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948446616, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1285214126110077, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50298880, "lr": 2.5579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948462117, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13054083287715912, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50462720, "lr": 2.5662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948462118, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948462118, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948556196, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.591177225112915, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948556197, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948556197, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 50462720, "epoch_num": 0.02202808612672653}} +:::MLLOG {"namespace": "", "time_ms": 1784948571038, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1221403181552887, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50626560, "lr": 2.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948586385, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12662363052368164, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50790400, "lr": 2.582916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948601651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12924233078956604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 50954240, "lr": 2.59125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948617051, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1329435557126999, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51118080, "lr": 2.5995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948633015, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13158658146858215, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51281920, "lr": 2.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948648447, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12167047709226608, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51445760, "lr": 2.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948664334, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1303580403327942, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51609600, "lr": 2.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948680318, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13210472464561462, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51773440, "lr": 2.6329166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948695625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13658763468265533, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 51937280, "lr": 2.64125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948710553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12807725369930267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52101120, "lr": 2.6495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948726367, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13305743038654327, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52264960, "lr": 2.6579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948742352, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11772496998310089, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52428800, "lr": 2.6662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948757856, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1212555319070816, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52592640, "lr": 2.6745833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948773452, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1256648600101471, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52756480, "lr": 2.6829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948773452, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948773453, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948868590, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6041976809501648, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948868591, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948868591, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 52756480, "epoch_num": 0.023029362768850464}} +:::MLLOG {"namespace": "", "time_ms": 1784948884280, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12165308743715286, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 52920320, "lr": 2.69125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948899465, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12752267718315125, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53084160, "lr": 2.6995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948915081, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1282796859741211, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53248000, "lr": 2.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948930379, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1315780133008957, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53411840, "lr": 2.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948945514, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216719001531601, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53575680, "lr": 2.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948961118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13438677787780762, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53739520, "lr": 2.7329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948976457, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12958930432796478, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 53903360, "lr": 2.74125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784948991574, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1216166689991951, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54067200, "lr": 2.7495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949007118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12687766551971436, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54231040, "lr": 2.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949022658, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1271541267633438, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54394880, "lr": 2.76625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949038149, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11860650777816772, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54558720, "lr": 2.774583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949054053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.13166259229183197, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54722560, "lr": 2.7829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949069364, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12672287225723267, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 54886400, "lr": 2.79125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949084774, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12824372947216034, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55050240, "lr": 2.799583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949084775, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949084775, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949178957, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6177582144737244, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949178958, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949178959, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 55050240, "epoch_num": 0.024030639410974396}} +:::MLLOG {"namespace": "", "time_ms": 1784949193669, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12841536104679108, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55214080, "lr": 2.807916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949209331, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1189737468957901, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55377920, "lr": 2.81625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949225088, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12190018594264984, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55541760, "lr": 2.824583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949240742, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12161259353160858, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55705600, "lr": 2.8329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949256238, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12383794784545898, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 55869440, "lr": 2.84125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949272003, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1265561580657959, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56033280, "lr": 2.849583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949287375, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11663893610239029, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56197120, "lr": 2.8579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949303000, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12298820167779922, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56360960, "lr": 2.86625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949318759, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12559285759925842, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56524800, "lr": 2.874583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949334096, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12388516217470169, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56688640, "lr": 2.8829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949349609, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11599884182214737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 56852480, "lr": 2.89125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949364977, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12472750246524811, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57016320, "lr": 2.899583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949380571, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11022903025150299, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57180160, "lr": 2.9079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949396346, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12448397278785706, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57344000, "lr": 2.91625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949396346, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949396347, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949491195, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6297577619552612, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949491196, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949491196, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 57344000, "epoch_num": 0.02503191605309833}} +:::MLLOG {"namespace": "", "time_ms": 1784949506966, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1172962561249733, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57507840, "lr": 2.924583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949522743, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11486797034740448, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57671680, "lr": 2.9329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949538261, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12929323315620422, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57835520, "lr": 2.94125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949554201, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12247244268655777, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 57999360, "lr": 2.9495833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949569859, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12469477951526642, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58163200, "lr": 2.9579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949585259, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11670318245887756, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58327040, "lr": 2.96625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949600058, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12050347775220871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58490880, "lr": 2.974583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949616391, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1222861111164093, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58654720, "lr": 2.9829166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949631974, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12163715064525604, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58818560, "lr": 2.99125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949647538, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11328402161598206, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 58982400, "lr": 2.9995833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949662802, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12176624685525894, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59146240, "lr": 3.0079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949678302, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12172585725784302, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59310080, "lr": 3.0162499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949693579, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12507115304470062, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59473920, "lr": 3.024583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949708932, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11723426729440689, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59637760, "lr": 3.0329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949708933, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949708933, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949803452, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6468431949615479, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949803453, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949803454, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 59637760, "epoch_num": 0.026033192695222262}} +:::MLLOG {"namespace": "", "time_ms": 1784949819060, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12069575488567352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59801600, "lr": 3.04125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949834066, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11184168606996536, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 59965440, "lr": 3.0495833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949849036, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11112887412309647, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60129280, "lr": 3.0579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949864428, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1139146089553833, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60293120, "lr": 3.0662499999999995e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949879572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10940233618021011, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60456960, "lr": 3.074583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949895160, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12187378108501434, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60620800, "lr": 3.082916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949910646, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11643083393573761, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60784640, "lr": 3.09125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949926400, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12039922177791595, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 60948480, "lr": 3.0995833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949941968, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10426948219537735, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61112320, "lr": 3.1079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949957846, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11994168162345886, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61276160, "lr": 3.1162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949973553, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11282042413949966, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61440000, "lr": 3.124583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784949989456, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340176314115524, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61603840, "lr": 3.132916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950004592, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10942406207323074, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61767680, "lr": 3.14125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950020527, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11699733138084412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 61931520, "lr": 3.1495833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950020528, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950020528, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950116062, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.6734912395477295, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950116063, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950116063, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 61931520, "epoch_num": 0.027034469337346195}} +:::MLLOG {"namespace": "", "time_ms": 1784950131814, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12016147375106812, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62095360, "lr": 3.1579166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950147429, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11987997591495514, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62259200, "lr": 3.1662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950163523, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12381482869386673, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62423040, "lr": 3.174583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950178996, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10693004727363586, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62586880, "lr": 3.1829166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950194760, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1140967383980751, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62750720, "lr": 3.19125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950210566, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11856260150671005, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 62914560, "lr": 3.1995833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950226231, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11955101042985916, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63078400, "lr": 3.207916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950242270, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11968287080526352, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63242240, "lr": 3.2162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950257823, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10903595387935638, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63406080, "lr": 3.2245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950273018, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11871703714132309, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63569920, "lr": 3.2329166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950288745, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11451846361160278, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63733760, "lr": 3.24125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950303695, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11709967255592346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 63897600, "lr": 3.2495833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950318969, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1134369894862175, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64061440, "lr": 3.257916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950334602, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247100979089737, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64225280, "lr": 3.26625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950334603, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950334603, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950429915, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7047087550163269, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950429916, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950429916, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 64225280, "epoch_num": 0.028035745979470127}} +:::MLLOG {"namespace": "", "time_ms": 1784950445625, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12371472269296646, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64389120, "lr": 3.2745833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950461437, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11717352271080017, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64552960, "lr": 3.2829166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950477120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11667104065418243, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64716800, "lr": 3.29125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950493049, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.121869757771492, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 64880640, "lr": 3.2995833333333336e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950509065, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10948721319437027, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65044480, "lr": 3.307916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950524522, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11145536601543427, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65208320, "lr": 3.31625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950539605, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1136884093284607, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65372160, "lr": 3.3245833333333334e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950555359, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1084410771727562, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65536000, "lr": 3.3329166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950571835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11108839511871338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65699840, "lr": 3.34125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950587940, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1159980520606041, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 65863680, "lr": 3.349583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950603889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12329971045255661, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66027520, "lr": 3.357916666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950619935, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11524996906518936, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66191360, "lr": 3.36625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950635816, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12680478394031525, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66355200, "lr": 3.3745833333333335e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950651903, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11369699984788895, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66519040, "lr": 3.3829166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950651903, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950651903, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950746438, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7452806830406189, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950746439, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950746439, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 66519040, "epoch_num": 0.02903702262159406}} +:::MLLOG {"namespace": "", "time_ms": 1784950762515, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12544560432434082, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66682880, "lr": 3.39125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950778120, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11340180039405823, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 66846720, "lr": 3.3995833333333327e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950794353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11583685874938965, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67010560, "lr": 3.4079166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950810310, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1131121814250946, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67174400, "lr": 3.41625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950825949, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11889799684286118, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67338240, "lr": 3.424583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950842512, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11722265928983688, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67502080, "lr": 3.4329166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950858660, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12064782530069351, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67665920, "lr": 3.44125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950874918, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12037031352519989, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67829760, "lr": 3.449583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950891338, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11503250151872635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 67993600, "lr": 3.4579166666666663e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950907785, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11171238124370575, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68157440, "lr": 3.46625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950923917, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11563924700021744, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68321280, "lr": 3.474583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950939651, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11449366807937622, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68485120, "lr": 3.4829166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950955127, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1247749850153923, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68648960, "lr": 3.49125e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950971569, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11532419919967651, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68812800, "lr": 3.499583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784950971570, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784950971570, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951065776, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7560091018676758, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951065776, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951065777, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 68812800, "epoch_num": 0.030038299263717993}} +:::MLLOG {"namespace": "", "time_ms": 1784951081685, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12148363888263702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 68976640, "lr": 3.5079166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951097927, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11821477860212326, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69140480, "lr": 3.51625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951113567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11334174871444702, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69304320, "lr": 3.524583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951129686, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11423928290605545, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69468160, "lr": 3.5329166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951145572, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10848351567983627, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69632000, "lr": 3.5412500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951161389, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11447238177061081, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69795840, "lr": 3.549583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951177554, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.117182157933712, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 69959680, "lr": 3.5579166666666664e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951193118, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11029712110757828, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70123520, "lr": 3.56625e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951209131, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10885175317525864, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70287360, "lr": 3.574583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951225208, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10804615914821625, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70451200, "lr": 3.582916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951241384, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.12229602038860321, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70615040, "lr": 3.5912500000000003e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951257835, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10594744980335236, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70778880, "lr": 3.599583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951273931, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.1201845109462738, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 70942720, "lr": 3.6079166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951290163, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11923522502183914, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71106560, "lr": 3.6162499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951290164, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951290164, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951384220, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7621324062347412, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951384221, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951384221, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 71106560, "epoch_num": 0.03103957590584193}} +:::MLLOG {"namespace": "", "time_ms": 1784951400567, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11783707141876221, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71270400, "lr": 3.624583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951416053, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11705782264471054, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71434240, "lr": 3.632916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951431393, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11529839038848877, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71598080, "lr": 3.6412499999999993e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951447770, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11519453674554825, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71761920, "lr": 3.649583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951463342, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10257124900817871, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 71925760, "lr": 3.6579166666666665e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951479405, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10244587063789368, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72089600, "lr": 3.6662499999999996e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951494938, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11874022334814072, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72253440, "lr": 3.674583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951510603, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11464724689722061, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72417280, "lr": 3.682916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951526313, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10299614816904068, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72581120, "lr": 3.6912499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951541889, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10176365822553635, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72744960, "lr": 3.699583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951557531, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10627223551273346, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 72908800, "lr": 3.7079166666666666e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951573106, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11212299019098282, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73072640, "lr": 3.7162499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951588667, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11058098077774048, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73236480, "lr": 3.7245833333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951604604, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11456134170293808, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73400320, "lr": 3.732916666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951604643, "event_type": "INTERVAL_END", "key": "block_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 278, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951604643, "event_type": "INTERVAL_START", "key": "eval_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 285, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951699547, "event_type": "POINT_IN_TIME", "key": "eval_accuracy", "value": 0.7663450837135315, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 326, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951699548, "event_type": "INTERVAL_END", "key": "eval_stop", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 329, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951699548, "event_type": "INTERVAL_START", "key": "block_start", "value": null, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/train/mlperf_logging_utils.py", "lineno": 271, "samples_count": 73400320, "epoch_num": 0.03204085254796586}} +:::MLLOG {"namespace": "", "time_ms": 1784951715177, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10499343276023865, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73564160, "lr": 3.7412499999999994e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951731306, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10615099221467972, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73728000, "lr": 3.749583333333333e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951747589, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.11413659155368805, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 73891840, "lr": 3.7579166666666667e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951763353, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10991502553224564, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74055680, "lr": 3.7662499999999997e-07}} +:::MLLOG {"namespace": "", "time_ms": 1784951779520, "event_type": "POINT_IN_TIME", "key": "train_loss", "value": 0.10793928056955338, "metadata": {"file": "/home/suachong/training/recommendation_v4/generative_recommenders/dlrm_v3/utils.py", "lineno": 1262, "samples_count": 74219520, "lr": 3.7745833333333333e-07}} diff --git a/recommendation_v4/requirements.txt b/recommendation_v4/requirements.txt new file mode 100644 index 000000000..852aa149b --- /dev/null +++ b/recommendation_v4/requirements.txt @@ -0,0 +1,43 @@ +# Frozen dependency versions for the recommendation_v4 (HSTU / yambda-5b) MLPerf +# reference. The CANONICAL, fully-reproducible environment is the Dockerfile +# (built on rocm/primus:v26.3); see docs/training_recipe.md for the per-platform +# (MI350X / B200) install commands and rationale. The pins below mirror that +# stack. torch / torchvision / torchaudio / fbgemm_gpu / torchrec are +# accelerator-specific and MUST be installed with --no-deps from the matching +# index (see Dockerfile) so pip does not clobber the +rocm wheels. + +# --- accelerator stack (install via Dockerfile; --no-deps, matching index) --- +# torch==2.12.0+rocm7.2 # --index-url https://download.pytorch.org/whl/rocm7.2 +# torchvision==0.27.0+rocm7.2 +# torchaudio==2.11.0+rocm7.2 +# fbgemm_gpu # built from FBGEMM commit 10b775730212923f65f7b78f79b6a01d80cf3c29 for gfx950 +torch==2.12.0 +fbgemm_gpu==1.7.0 +torchrec @ git+https://github.com/pytorch/torchrec.git@v2026.06.01.00 + +# --- data / config / logging ------------------------------------------------- +polars-u64-idx==1.33.1 +gin-config==0.5.0 +absl-py==2.1.0 +pandas==2.2.3 +pyarrow==17.0.0 +numpy==1.26.4 +xxhash==3.5.0 +datasets==3.2.0 +huggingface_hub==0.27.0 + +# --- metrics / training utils ------------------------------------------------ +torchmetrics==1.0.3 +tensordict==0.6.2 +tensorboard==2.19.0 +pyre-extensions==0.0.32 +iopath==0.1.10 +typing-inspect==0.9.0 +psutil==6.1.0 +tqdm==4.67.1 +pyyaml==6.0.2 +pybind11==2.13.6 +lightning-utilities==0.11.9 + +# --- MLPerf compliance logging (pinned to the Training 6.0 tag) -------------- +mlperf-logging @ git+https://github.com/mlcommons/logging.git@6.0.0-rc6 diff --git a/recommendation_v4/run_and_time.sh b/recommendation_v4/run_and_time.sh new file mode 100755 index 000000000..a0207e71c --- /dev/null +++ b/recommendation_v4/run_and_time.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# MLPerf Training reference script: run the benchmark and report wall-clock time. +# +# Runs the full-reference HSTU / yambda-5b streaming train+eval sweep to the +# MLPerf quality target (eval AUC >= 0.80275) and prints the elapsed time of the +# timed region. This is the canonical single-host (8-GPU) entry point; for +# multi-node SLURM launches use scripts/launch_slurm.sh (which calls into the +# same trainer). +# +# Usage: +# DLRM_DATA_PATH=/path/to/dlrm_data ./run_and_time.sh +# +# Env (run shape / cadence -- defaults are the FULL reference sweep): +# DLRM_DATA_PATH data root (required). +# SEED RNG seed (default 1). +# START_TS / NUM_TRAIN_TS window range (default 0 / 299 = full sweep). +# EVAL_EVERY_DATA_PCT eval cadence as a fraction of train data (default 0.005). +# AUC_THRESHOLD convergence target (default 0.80275). +# GPUS_PER_NODE GPUs on this host (default 8). +# RUN_NAME results dir name under results/ (default reference_run). +set -euo pipefail + +: "${DLRM_DATA_PATH:?Set DLRM_DATA_PATH to the data root}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "${REPO_ROOT}" + +# ---- Reference run shape (full sweep to the quality target) ----------------- +export SEED="${SEED:-1}" +export START_TS="${START_TS:-0}" +export NUM_TRAIN_TS="${NUM_TRAIN_TS:-299}" +export NUM_TRAIN_BATCHES="${NUM_TRAIN_BATCHES:-0}" +export NUM_EVAL_BATCHES="${NUM_EVAL_BATCHES:-0}" +export EVAL_EVERY_N_WINDOWS="${EVAL_EVERY_N_WINDOWS:-0}" +export EVAL_EVERY_DATA_PCT="${EVAL_EVERY_DATA_PCT:-0.005}" +export AUC_THRESHOLD="${AUC_THRESHOLD:-0.80275}" +export RUN_NAME="${RUN_NAME:-reference_run}" + +# ---- Single-host distributed topology (override for multi-node) ------------- +export GPUS_PER_NODE="${GPUS_PER_NODE:-8}" +export NNODES="${NNODES:-1}" +export NODE_RANK="${NODE_RANK:-0}" +export WORLD_SIZE="${WORLD_SIZE:-$((NNODES * GPUS_PER_NODE))}" +export MASTER_ADDR="${MASTER_ADDR:-127.0.0.1}" +export MASTER_PORT="${MASTER_PORT:-29500}" + +# ---- MLPerf compliance logging ---------------------------------------------- +export MLPERF_LOGGING="${MLPERF_LOGGING:-1}" +export MLPERF_LOG_PATH="${MLPERF_LOG_PATH:-${REPO_ROOT}/results/${RUN_NAME}/mlperf/yambda_5b_mlperf.log}" +export MLPERF_SUBMISSION_PLATFORM="${MLPERF_SUBMISSION_PLATFORM:-MI355X}" +mkdir -p "$(dirname "${MLPERF_LOG_PATH}")" + +# ---- Timed region ----------------------------------------------------------- +# Pull the start timestamp into a clear region per the MLPerf run_and_time.sh idiom. +start=$(date +%s) +echo "STARTING TIMING RUN AT $(date -u '+%Y-%m-%d %r')" + +python -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset yambda-5b \ + --mode streaming-train-eval + +end=$(date +%s) +result=$(( end - start )) +echo "ENDING TIMING RUN AT $(date -u '+%Y-%m-%d %r')" +echo "RESULT,recommendation_v4_hstu_yambda_5b,${SEED},${result},$(whoami),$(date -u '+%Y-%m-%d %r')" diff --git a/recommendation_v4/scripts/build_ne_auc_trajectory.py b/recommendation_v4/scripts/build_ne_auc_trajectory.py new file mode 100644 index 000000000..edc57cfbf --- /dev/null +++ b/recommendation_v4/scripts/build_ne_auc_trajectory.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Build a combined train+eval NE/AUC trajectory from a streaming-train-eval log. + +The streaming loop (generative_recommenders/dlrm_v3/train/utils.py) emits, via +MetricsLogger.compute(), one line per logged step of the form: + + INFO:utils:train - Step 201 metrics: {'metric/lifetime_ne/listen_plus': + tensor(1.0182, dtype=torch.float64), 'metric/window_ne/listen_plus': + tensor(0.9846, ...), ..., 'metric/window_auc/listen_plus': tensor(0.5912), + 'metric/lifetime_auc/listen_plus': tensor(0.5480)} + +and the analogous `eval - Step N metrics:` lines during each (full-holdout) eval +window, plus throughput lines: + + INFO:utils:train - Step 201 perf: local_sps=97.0 global_sps=776.2 + step_ms=10553.89 elapsed_sec=680.6 total_samples=205824 + +This script parses all three, for a chosen task (default listen_plus), and writes: + * /trajectory.json — {"train": {step: {...}}, "eval": {...}, "perf": [...]} + * /trajectory.csv — long-form rows (mode, step, metric, value) + * /trajectory_ne_auc.png — NE and AUC vs train step, train + eval overlaid + (skipped gracefully if matplotlib is absent) + +It is dependency-light (stdlib + optional matplotlib) so it runs anywhere the +log is readable, including the head node. + +Usage: + python3 scripts/build_ne_auc_trajectory.py LOG [--out DIR] [--task listen_plus] +""" + +import argparse +import csv +import json +import os +import re +import sys +from typing import Dict, List, Optional, Tuple + +# `train - Step 201 metrics: {...}` / `eval - Step 17 metrics: {...}` +_STEP_RE = re.compile(r"(train|eval) - Step (\d+) metrics: \{(.*)\}") +# `metric//': tensor(` — value may be int/float/sci. +_METRIC_RE = re.compile( + r"metric/([A-Za-z0-9_]+)/([A-Za-z0-9_+]+)'?\s*:\s*tensor\(\s*([-0-9.eE+]+)" +) +# `train - Step 201 perf: local_sps=97.0 global_sps=776.2 step_ms=10553.89 ` +# `elapsed_sec=680.6 total_samples=205824` +_PERF_RE = re.compile( + r"train - Step (\d+) perf: local_sps=([-0-9.eE+]+) global_sps=([-0-9.eE+]+) " + r"step_ms=([-0-9.eE+]+) elapsed_sec=([-0-9.eE+]+) total_samples=(\d+)" +) +# `[boundary] eval_ts=181 eval first-batch ...` — marks the start of a full-holdout +# eval block; the eval runs at whatever the latest train global step was, so we use +# it to anchor each eval's metrics onto the shared train-global-step x-axis. +_EVAL_BOUNDARY_RE = re.compile(r"\[boundary\] eval_ts=(\d+) eval first-batch") + +# Metrics we surface in the trajectory (others are still captured if present). +_KEEP = ("window_ne", "lifetime_ne", "window_auc", "lifetime_auc", + "window_accuracy", "lifetime_accuracy", "window_gauc", "lifetime_gauc") + + +def _parse_metrics(body: str, task: str) -> Dict[str, float]: + row: Dict[str, float] = {} + for name, tname, val in _METRIC_RE.findall(body): + if tname != task: + continue + try: + row[name] = float(val) + except ValueError: + continue + return row + + +def parse_log( + log_path: str, task: str +) -> Tuple[Dict[str, Dict[int, Dict[str, float]]], List[Dict[str, float]]]: + """Return ({'train': {step: {metric: val}}, 'eval': {...}}, perf_rows). + + Train is keyed by train global step (last write wins — duplicate per-rank + prints are identical). Eval uses a per-rank-resetting internal step counter + that restarts every eval window, so we instead anchor each eval window onto + the *train global step at which it ran* (the loop trains window T then evals + window T+1, so the eval's anchor is the last train step before it). Each eval + window collapses to a single point carrying its final, most-aggregated + full-holdout metrics, plus `eval_window` (the eval_ts) for reference. + """ + out: Dict[str, Dict[int, Dict[str, float]]] = {"train": {}, "eval": {}} + perf: List[Dict[str, float]] = [] + + last_train_step = 0 + cur_anchor: Optional[int] = None # train global step this eval block runs at + cur_ts: Optional[int] = None # eval window id (eval_ts) + cur_row: Optional[Dict[str, float]] = None # final row of the current block + cur_internal: Optional[int] = None # last eval internal step (reset detection) + + def flush_eval() -> None: + nonlocal cur_anchor, cur_ts, cur_row, cur_internal + if cur_row: + anchor = cur_anchor if cur_anchor is not None else last_train_step + row = dict(cur_row) + if cur_ts is not None: + row["eval_window"] = float(cur_ts) + key = anchor + while key in out["eval"]: # keep distinct evals from colliding + key += 1 + out["eval"][key] = row + cur_anchor = cur_ts = cur_row = cur_internal = None + + with open(log_path, "r", errors="replace") as f: + for line in f: + pm = _PERF_RE.search(line) + if pm: + perf.append({ + "step": int(pm.group(1)), + "local_sps": float(pm.group(2)), + "global_sps": float(pm.group(3)), + "step_ms": float(pm.group(4)), + "elapsed_sec": float(pm.group(5)), + "total_samples": int(pm.group(6)), + }) + continue + bm = _EVAL_BOUNDARY_RE.search(line) + if bm: + # The boundary line (a different logger) can interleave before OR + # after this eval's metric lines, so don't use it to delimit the + # block — just tag the current block with its eval_ts. Block + # boundaries come from eval-step resets / training resuming. + if cur_anchor is None: + cur_anchor = last_train_step + cur_ts = int(bm.group(1)) + continue + m = _STEP_RE.search(line) + if not m: + continue + mode, step_s, body = m.group(1), m.group(2), m.group(3) + step = int(step_s) + row = _parse_metrics(body, task) + if mode == "train": + last_train_step = step + if cur_anchor is not None or cur_row is not None: + flush_eval() # an eval block ends when training resumes + if row: + out["train"][step] = row # last write wins + else: # eval — accumulate into the current block (last = most aggregated) + # Fallback for logs without a boundary marker: a drop in the eval + # internal step counter signals a fresh eval window. + if (cur_internal is not None and step < cur_internal + and cur_anchor is None): + flush_eval() + if cur_anchor is None: + cur_anchor = last_train_step + cur_internal = step + if row: + cur_row = row + flush_eval() + return out, perf + + +def write_outputs( + traj: Dict[str, Dict[int, Dict[str, float]]], + perf: List[Dict[str, float]], + out_dir: str, + task: str, +) -> None: + os.makedirs(out_dir, exist_ok=True) + + json_path = os.path.join(out_dir, "trajectory.json") + with open(json_path, "w") as f: + json.dump( + { + "task": task, + "train": {str(k): v for k, v in sorted(traj["train"].items())}, + "eval": {str(k): v for k, v in sorted(traj["eval"].items())}, + "perf": perf, + }, + f, + indent=2, + ) + + csv_path = os.path.join(out_dir, "trajectory.csv") + with open(csv_path, "w", newline="") as f: + w = csv.writer(f) + w.writerow(["mode", "step", "metric", "value"]) + for mode in ("train", "eval"): + for step in sorted(traj[mode]): + for metric, val in traj[mode][step].items(): + w.writerow([mode, step, metric, val]) + + n_train = len(traj["train"]) + n_eval = len(traj["eval"]) + print(f"Parsed {n_train} train points, {n_eval} eval points, " + f"{len(perf)} perf points (task={task}).", file=sys.stderr) + print(f"Wrote {json_path}", file=sys.stderr) + print(f"Wrote {csv_path}", file=sys.stderr) + + _maybe_plot(traj, out_dir, task) + + +def _maybe_plot( + traj: Dict[str, Dict[int, Dict[str, float]]], out_dir: str, task: str +) -> None: + try: + import matplotlib + matplotlib.use("Agg") + import matplotlib.pyplot as plt + except Exception as e: # noqa: BLE001 + print(f"matplotlib unavailable ({e}); skipping plot.", file=sys.stderr) + return + + def series(mode: str, metric: str) -> Tuple[List[int], List[float]]: + steps = sorted(s for s in traj[mode] if metric in traj[mode][s]) + return steps, [traj[mode][s][metric] for s in steps] + + fig, (ax_ne, ax_auc) = plt.subplots(2, 1, figsize=(11, 9), sharex=True) + + for metric, style in (("window_ne", "-"), ("lifetime_ne", "--")): + xs, ys = series("train", metric) + if xs: + ax_ne.plot(xs, ys, style, label=f"train/{metric}", alpha=0.85) + for metric, marker in (("window_ne", "o"), ("lifetime_ne", "s")): + xs, ys = series("eval", metric) + if xs: + ax_ne.plot(xs, ys, marker=marker, ms=5, ls="-", lw=1.0, alpha=0.9, + label=f"eval/{metric}") + ax_ne.set_ylabel("NE (normalized entropy)") + ax_ne.set_title(f"yambda-5b streaming train+eval trajectory — task={task}") + ax_ne.grid(True, alpha=0.3) + ax_ne.legend(fontsize=8, ncol=2) + + for metric, style in (("window_auc", "-"), ("lifetime_auc", "--")): + xs, ys = series("train", metric) + if xs: + ax_auc.plot(xs, ys, style, label=f"train/{metric}", alpha=0.85) + for metric, marker in (("window_auc", "o"), ("lifetime_auc", "s")): + xs, ys = series("eval", metric) + if xs: + ax_auc.plot(xs, ys, marker=marker, ms=5, ls="-", lw=1.0, alpha=0.9, + label=f"eval/{metric}") + ax_auc.set_ylabel("AUC") + ax_auc.set_xlabel("train global step (eval points anchored to the step they ran at)") + ax_auc.grid(True, alpha=0.3) + ax_auc.legend(fontsize=8, ncol=2) + + png_path = os.path.join(out_dir, "trajectory_ne_auc.png") + fig.tight_layout() + fig.savefig(png_path, dpi=120) + print(f"Wrote {png_path}", file=sys.stderr) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("log", help="Path to the streaming train.log") + ap.add_argument("--out", default=None, + help="Output dir (default: /_trajectory)") + ap.add_argument("--task", default="listen_plus", + help="Task name to extract (default: listen_plus)") + args = ap.parse_args() + + if not os.path.exists(args.log): + print(f"Log not found: {args.log}", file=sys.stderr) + return 2 + out_dir = args.out + if out_dir is None: + stem = os.path.splitext(os.path.basename(args.log))[0] + out_dir = os.path.join(os.path.dirname(os.path.abspath(args.log)), + f"{stem}_trajectory") + + traj, perf = parse_log(args.log, args.task) + write_outputs(traj, perf, out_dir, args.task) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/recommendation_v4/scripts/cv/steps_to_target_cv.py b/recommendation_v4/scripts/cv/steps_to_target_cv.py new file mode 100644 index 000000000..465c48fd4 --- /dev/null +++ b/recommendation_v4/scripts/cv/steps_to_target_cv.py @@ -0,0 +1,174 @@ +#!/usr/bin/env python3 +"""Run-to-run convergence CV via the steps-to-target method. + +For each target eval-window AUC t, for each run find the first samples_count +where AUC >= t (first-crossing, no interpolation). Report mean/std/CV/min/max +across the runs that reached t. + +Reads MLPerf ``:::MLLOG`` logs (``eval_accuracy`` == window_auc/listen_plus). + +Examples:: + + # auto sweep 0.60 .. max-AUC @ step 0.005 + python3 scripts/cv/steps_to_target_cv.py --log-dir /path/to/logs -o cv/table.csv + + # single target + python3 scripts/cv/steps_to_target_cv.py --log-dir /path/to/logs --target 0.75 + + # explicit list + python3 scripts/cv/steps_to_target_cv.py --log-dir /path/to/logs --targets 0.70,0.75,0.78 + + # range with step + python3 scripts/cv/steps_to_target_cv.py --log-dir /path/to/logs --target-range 0.60:0.78 --step 0.005 +""" +import argparse +import glob +import json +import os +import re +import sys + +import numpy as np +import pandas as pd + +_MLLOG = re.compile(r":::MLLOG (\{.*\})\s*$") + + +def parse_log(path): + """MLPerf log -> (samples_sorted, auc_sorted). Empty arrays if no evals.""" + pts = {} + with open(path, "r", errors="replace") as f: + for line in f: + m = _MLLOG.search(line) + if not m: + continue + try: + rec = json.loads(m.group(1)) + except json.JSONDecodeError: + continue + if rec.get("key") != "eval_accuracy" or rec.get("value") is None: + continue + sc = rec.get("metadata", {}).get("samples_count") + if sc is not None: + pts[int(sc)] = float(rec["value"]) + if not pts: + return np.array([]), np.array([]) + samples = np.array(sorted(pts)) + auc = np.array([pts[s] for s in samples]) + return samples, auc + + +def load_runs(log_dir): + """All *.log files in log_dir -> {basename: (samples, auc)}.""" + paths = sorted(glob.glob(os.path.join(log_dir, "*.log"))) + if not paths: + sys.exit(f"no *.log files in {log_dir}") + runs = {} + for path in paths: + name = os.path.splitext(os.path.basename(path))[0] + samples, auc = parse_log(path) + if len(auc) >= 1: + runs[name] = (samples, auc) + print(f" {name}: {len(auc)} eval points", file=sys.stderr) + return runs + + +def first_crossing(samples, auc, target): + for s, a in zip(samples, auc): + if a >= target: + return s + return None + + +def resolve_targets(args, runs): + """Build target AUC list from CLI flags.""" + if args.target is not None: + return [float(args.target)] + if args.targets is not None: + return [float(t.strip()) for t in args.targets.split(",") if t.strip()] + + lo, hi = args.target_range + top = max(a.max() for _, a in runs.values()) + hi = min(hi, top) + if lo > hi: + sys.exit(f"target-range lo={lo} > hi={hi} (max AUC across runs = {top:.4f})") + return list(np.arange(lo, hi + 1e-9, args.step)) + + +def cv_table(runs, targets, min_runs): + rows = [] + for t in targets: + hits = [s for samples, auc in runs.values() + if (s := first_crossing(samples, auc, t)) is not None] + n = len(hits) + if n < min_runs: + continue + arr = np.array(hits, dtype=float) + mean = arr.mean() + std = arr.std(ddof=1) + rows.append({ + "target_eval_auc": round(float(t), 4), + "runs_reached": n, + "samples_mean": round(mean, 1), + "samples_std": round(std, 1), + "cv_pct": round(std / mean * 100.0, 3) if mean > 0 else 0.0, + "cv_fraction": round(std / mean, 5) if mean > 0 else 0.0, + "samples_min": int(arr.min()), + "samples_max": int(arr.max()), + }) + return pd.DataFrame(rows) + + +def parse_target_range(s): + if ":" not in s: + raise argparse.ArgumentTypeError("expected LO:HI, e.g. 0.60:0.78") + lo, hi = s.split(":", 1) + return float(lo), float(hi) + + +def main(): + p = argparse.ArgumentParser( + description="Steps-to-target CV table from a folder of MLPerf logs.") + p.add_argument("--log-dir", required=True, + help="directory containing one *.log per seed run") + p.add_argument("-o", "--output", + help="output CSV path (default: /steps_to_target_cv.csv)") + p.add_argument("--min-runs", type=int, default=2, + help="min runs that must reach a target to include it (default: 2)") + + g = p.add_mutually_exclusive_group() + g.add_argument("--target", type=float, + help="single target eval-window AUC, e.g. 0.75") + g.add_argument("--targets", metavar="T1,T2,...", + help="comma-separated target AUCs, e.g. 0.70,0.75,0.78") + g.add_argument("--target-range", type=parse_target_range, metavar="LO:HI", + help="inclusive AUC range, e.g. 0.60:0.78 (default when unset)") + + p.add_argument("--step", type=float, default=0.005, + help="step for --target-range sweep (default: 0.005)") + p.add_argument("--range-lo", type=float, default=0.60, + help="lower bound when no target flag given (default: 0.60)") + args = p.parse_args() + + if args.target_range is None and args.target is None and args.targets is None: + args.target_range = (args.range_lo, 1.0) # hi clipped to max AUC in resolve_targets + + print(f"loading logs from {args.log_dir}", file=sys.stderr) + runs = load_runs(args.log_dir) + if len(runs) < args.min_runs: + sys.exit(f"only {len(runs)} usable run(s) in {args.log_dir} (need >= {args.min_runs})") + + targets = resolve_targets(args, runs) + df = cv_table(runs, targets, args.min_runs) + if df.empty: + sys.exit("no targets had enough runs — check --target / --target-range / --min-runs") + + out = args.output or os.path.join(args.log_dir, "steps_to_target_cv.csv") + os.makedirs(os.path.dirname(os.path.abspath(out)) or ".", exist_ok=True) + df.to_csv(out, index=False) + print(f"wrote {out} ({len(runs)} runs, {len(df)} targets)", file=sys.stderr) + print(df.to_string(index=False)) + + +if __name__ == "__main__": + main() diff --git a/recommendation_v4/scripts/launch_local.sh b/recommendation_v4/scripts/launch_local.sh new file mode 100755 index 000000000..45f4bb6f2 --- /dev/null +++ b/recommendation_v4/scripts/launch_local.sh @@ -0,0 +1,157 @@ +#!/bin/bash +# ============================================================================= +# launch_local.sh — single-host, NON-SLURM launcher for the yambda-5b trainer. +# +# This is the SLURM-free analog of scripts/launch_slurm.sh's `worker` phase: +# it sets the single-node distributed topology + sane env and invokes the SAME +# entry point (`train_ranker.py --dataset yambda-5b`) reading the SAME +# train/gin/yambda_5b.gin config. No scheduler, no docker, no RDMA overlay — +# everything runs directly on this host against an already-prepared dataset. +# +# Use it to: +# * Smoke-test the launch path on a single GPU box (SMOKE=1, the default — +# a few train/eval batches of one streaming window), or +# * Run the full gin-default workload (SMOKE=0 — consumes whole windows). +# +# PREREQUISITES +# 1) Data prepared (run once, CPU-only — no GPU needed): +# python generative_recommenders/dlrm_v3/preprocess_public_data.py \ +# --dataset yambda-5b --data-path "$DLRM_DATA_PATH" +# producing $DLRM_DATA_PATH/processed_5b/{train_sessions.parquet,...} +# and $DLRM_DATA_PATH/shared_metadata/{artist,album}_item_mapping.parquet +# 2) The train_recipe GPU stack importable by $PYTHON (see docs/training_recipe.md): +# torch (rocm or cuda build), fbgemm_gpu, torchrec, polars-u64-idx, +# gin-config, xxhash, pandas, tensorboard, ... +# This box must have visible GPUs (the trainer shards embeddings onto HBM). +# +# USAGE +# # smoke (default): one window, 20 train + 10 eval batches +# DLRM_DATA_PATH=/home/chcai/dlrm_data bash scripts/launch_local.sh +# +# # full gin-default run (whole windows; long) +# SMOKE=0 DLRM_DATA_PATH=/home/chcai/dlrm_data bash scripts/launch_local.sh +# +# # restrict to 2 GPUs, custom log, plain (non-streaming) train-eval +# GPUS_PER_NODE=2 MODE=train-eval LOG=/tmp/y.log bash scripts/launch_local.sh +# +# Every knob below is env-overridable; defaults reproduce launch_slurm.sh's +# single-node smoke path so a local run matches the known-good cluster path. +# ============================================================================= +set -uo pipefail + +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) +cd "$REPO_ROOT" + +# ---- interpreter ------------------------------------------------------------ +# Default to the venv created for data prep if present, else system python3. +# Override with PYTHON=/path/to/python (e.g. the in-container recipe python). +DEFAULT_PY=/home/chcai/dlrmv4_venv/bin/python +PYTHON=${PYTHON:-$([ -x "$DEFAULT_PY" ] && echo "$DEFAULT_PY" || echo python3)} + +# ---- dataset / data path ---------------------------------------------------- +DATASET=${DATASET:-yambda-5b} +MODE=${MODE:-streaming-train-eval} +# Mirrors the yambda_5b.gin default ("/apps/chcai/dlrm_data"); point at wherever +# preprocess_public_data.py wrote processed_5b/ + shared_metadata/. +export DLRM_DATA_PATH=${DLRM_DATA_PATH:-/home/chcai/dlrm_data} + +LOG=${LOG:-$REPO_ROOT/yambda_local.$(date +%Y%m%d_%H%M%S).log} + +# ---- single-node distributed topology -------------------------------------- +# train_ranker reads these from the env (see train_ranker.main): it spawns +# GPUS_PER_NODE ranks via torch.multiprocessing on THIS host. localhost +# rendezvous; empty MASTER_PORT => train_ranker picks a free port. +export NNODES=${NNODES:-1} +export NODE_RANK=${NODE_RANK:-0} +export MASTER_ADDR=${MASTER_ADDR:-localhost} +export MASTER_PORT=${MASTER_PORT:-} +# GPUS_PER_NODE: 0/unset => train_ranker auto-detects torch.cuda.device_count(). +export GPUS_PER_NODE=${GPUS_PER_NODE:-0} + +# ---- runtime env (matches launch_slurm.sh worker defaults) ------------------ +export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-TRITON} +export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} +export PYTHONPATH="$REPO_ROOT:${PYTHONPATH:-}" +# Single-node RCCL bootstrap: all ranks rendezvous over localhost, so pin the +# loopback NIC. Left to auto-detect, RCCL can grab a non-routable per-GPU RoCE +# NIC and hang/"No route to host" at init (same failure launch_slurm.sh pins +# fenic0 to avoid). Override NCCL_SOCKET_IFNAME for a routable multi-host setup. +export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-lo} +export NCCL_DEBUG=${NCCL_DEBUG:-WARN} + +# ---- smoke caps ------------------------------------------------------------- +# SMOKE=1 (default): apply small per-window batch caps so a launch finishes in +# minutes (validates the path end-to-end). SMOKE=0: leave the gin defaults +# untouched (consume full windows — the real workload). +SMOKE=${SMOKE:-1} +if [ "$SMOKE" = "1" ]; then + export START_TS=${START_TS:-150} + export NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} + export NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} + export NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} + # Default eval cadence: per-window OFF (0), data-fraction every 0.5% of data + # (0.005). Mutually exclusive (both >0 raises a ValueError at startup). + export EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-0} + export EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0.005} + export METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} + # Smaller per-sample shape keeps the smoke run light; drop these to use the + # gin defaults (4086/4096). Reuse an existing hstu_cache_L/ if present. + export BATCH_SIZE=${BATCH_SIZE:-32} +fi + +mkdir -p "$(dirname "$LOG")" +{ + echo "[$(date)] launch_local: dataset=$DATASET mode=$MODE smoke=$SMOKE" + echo "[$(date)] PYTHON=$PYTHON" + echo "[$(date)] DLRM_DATA_PATH=$DLRM_DATA_PATH" + echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node(req)=$GPUS_PER_NODE master=$MASTER_ADDR:${MASTER_PORT:-}" +} | tee -a "$LOG" + +# ---- preflight: data present? ---------------------------------------------- +SUFFIX=${DATASET#yambda-} +PROCESSED="$DLRM_DATA_PATH/processed_${SUFFIX}/train_sessions.parquet" +META="$DLRM_DATA_PATH/shared_metadata/artist_item_mapping.parquet" +if [ "$DATASET" = "yambda-5b" ] && { [ ! -f "$PROCESSED" ] || [ ! -f "$META" ]; }; then + echo "[$(date)] ERROR: prepared data not found." | tee -a "$LOG" + echo " expected: $PROCESSED" | tee -a "$LOG" + echo " and: $META" | tee -a "$LOG" + echo " run preprocessing first:" | tee -a "$LOG" + echo " $PYTHON generative_recommenders/dlrm_v3/preprocess_public_data.py --dataset $DATASET --data-path $DLRM_DATA_PATH" | tee -a "$LOG" + exit 1 +fi + +# ---- preflight: GPU stack importable + GPUs visible? ------------------------ +echo "[$(date)] preflight: checking torch / fbgemm_gpu / torchrec + GPU count" | tee -a "$LOG" +"$PYTHON" - <<'PY' 2>&1 | tee -a "$LOG" +import sys +missing = [] +for m in ("torch", "fbgemm_gpu", "torchrec", "polars", "gin", "xxhash"): + try: + __import__(m) + except Exception as e: + missing.append(f"{m} ({e.__class__.__name__})") +if missing: + print("PREFLIGHT FAIL: missing/broken imports: " + ", ".join(missing)) + print("Install the train_recipe GPU stack (see docs/training_recipe.md).") + sys.exit(3) +import torch +n = torch.cuda.device_count() +print(f"imports OK, torch {torch.__version__}, cuda/hip available={torch.cuda.is_available()}, {n} GPU(s)") +if n == 0: + print("PREFLIGHT FAIL: no GPUs visible — the HSTU trainer shards embeddings " + "onto GPU HBM and cannot run CPU-only. Launch on a GPU host.") + sys.exit(4) +PY +pf=${PIPESTATUS[0]} +if [ "$pf" -ne 0 ]; then + echo "[$(date)] preflight failed (rc=$pf) — not launching trainer." | tee -a "$LOG" + exit "$pf" +fi + +# ---- launch ----------------------------------------------------------------- +echo "[$(date)] launching train_ranker ($DATASET, mode=$MODE)" | tee -a "$LOG" +"$PYTHON" -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset "$DATASET" --mode "$MODE" 2>&1 | tee -a "$LOG" +rc=${PIPESTATUS[0]} +echo "[$(date)] launch_local finished rc=$rc" | tee -a "$LOG" +exit "$rc" diff --git a/recommendation_v4/scripts/launch_slurm.sh b/recommendation_v4/scripts/launch_slurm.sh new file mode 100755 index 000000000..b1289f144 --- /dev/null +++ b/recommendation_v4/scripts/launch_slurm.sh @@ -0,0 +1,667 @@ +#!/bin/bash +#SBATCH --job-name=yambda_slurm +#SBATCH --nodes=1 +#SBATCH --ntasks-per-node=1 +#SBATCH --exclusive +#SBATCH --partition=meta64 # [CLUSTER-SPECIFIC] partition name +#SBATCH --output=/apps/chcai/yambda_slurm.%j.out +# ============================================================================= +# launch_slurm.sh — single entry point for the yambda-5b trainer on N>=1 nodes. +# +# Consolidates what used to be separate scripts so multi-node enablement is +# ONE committable script (plus the train_ranker.py / utils.py python changes): +# * orchestrate phase (host SLURM glue) — formerly sbatch_smoke_multinode.sh +# * provision phase (container + RDMA) — formerly _provision_yambda_primus.sh +# * worker phase (in-container train) — now inlined below +# +# PHASES (auto-detected from context; force with LAUNCH_SLURM_PHASE=): +# orchestrate Runs on the SLURM batch host (no /.dockerenv). Resolves the +# rendezvous (MASTER_ADDR/PORT), ensures the container on every +# node (provision phase), then `docker exec`s the worker phase on +# every node, one task per node. +# provision Runs on a compute-node host. Ensures the `yambda_primus` +# container is up (loads the pre-baked image if present — no +# internet/pip — else builds from the base image) and stages the +# host RDMA userspace overlay on shared NFS. +# worker Runs INSIDE the container. Sets the distributed topology + +# NCCL/RDMA env and spawns this node's GPU ranks via train_ranker. +# N==1 transparently uses the legacy single-node path (localhost, +# node_rank 0), byte-for-byte as before, so the streaming-e2e +# supervisor's direct `bash scripts/launch_slurm.sh` is unchanged. +# +# USAGE +# Multi-node (N>=1): sbatch --nodes=2 scripts/launch_slurm.sh +# Single-node direct: bash scripts/launch_slurm.sh (already inside container; +# what run_streaming_e2e.sh invokes per relaunch) +# Perf pair: +# LOG=/apps/chcai/perf_1node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ +# EVAL_EVERY_N_WINDOWS=0 METRIC_LOG_FREQ=20 \ +# sbatch --nodes=1 --job-name=y1 scripts/launch_slurm.sh +# LOG=/apps/chcai/perf_2node.log NUM_TRAIN_BATCHES=200 NUM_EVAL_BATCHES=0 \ +# EVAL_EVERY_N_WINDOWS=0 METRIC_LOG_FREQ=20 \ +# sbatch --nodes=2 --job-name=y2 scripts/launch_slurm.sh +# # then: bash scripts/compare_node_perf.sh /apps/chcai/perf_1node.log /apps/chcai/perf_2node.log +# +# ONE-TIME IMAGE BAKE (so fresh nodes skip the multi-GB torch download + pip): +# BAKE_IMAGE=1 LAUNCH_SLURM_PHASE=provision bash scripts/launch_slurm.sh +# (commits the deps-installed container to $BAKED_IMAGE and `docker save`s it to +# $BAKED_TAR on NFS; subsequent provisions `docker load` it offline.) +# +# ----------------------------------------------------------------------------- +# PORTABILITY — what to change for a DIFFERENT cluster / network / hardware. +# Every such knob is also tagged inline with "[CLUSTER-SPECIFIC]" (grep for it). +# All are env-overridable, so you can adapt without editing this file. +# +# A) SLURM / scheduler +# - #SBATCH --partition=meta64 : partition name. CHANGE per cluster. +# - #SBATCH --time / --exclusive : policy; adjust to taste. +# +# B) Filesystems (must be shared/NFS across ALL nodes — this script re-invokes +# itself and reads the overlay + data from these paths cluster-wide) +# - REPO_MOUNT (repo + this script, e.g. /home/) is bind-mounted rw; +# DATA_MOUNT (e.g. /apps/chcai) holds the read-only dataset + overlay + +# baked tar + pip tarball; SCRATCH (e.g. /home//yambda_runs) is the +# writable log/output root. Override any via env — nothing is user-hardwired. +# +# C) Container image / GPU software stack (tied to the GPU arch + ROCm version) +# - IMAGE=rocm/primus:v26.3 : base image. ROCm/AMD-specific. +# - docker run --device=/dev/kfd --device=/dev/dri --group-add video : AMD ROCm +# device passthrough. For NVIDIA this is --gpus all / nvidia runtime instead. +# - --ulimit memlock=-1 : REQUIRED for RDMA QP registration (do not drop). +# - TORCH_IDX (rocm7.2), torch/vision/audio ==*+rocm7.2, FBGEMM_WHL (a gfx950 +# wheel), torchrec pin : the whole deps set is arch/ROCm-version-specific. +# +# D) Network fabric — THE trickiest part; defaults are PROVEN on meta64 cv350 +# (Broadcom bnxt_re RoCEv2). On a different fabric these almost certainly change +# (see the worker-phase block for the full rationale): +# - NCCL_SOCKET_IFNAME=fenic0 : the ONE routable host NIC for TCP bootstrap. +# Find yours with `ip -br addr`; the per-GPU RDMA NICs are usually NOT +# routable for plain TCP, so auto-detect hangs init — you MUST pin this. +# - NCCL_IB_HCA=bnxt_re0..7 : the RDMA HCA device names. List with `ibv_devices`. +# Different NIC vendor (e.g. mlx5_*, ionic_*) => different names AND a +# different userspace provider, which changes the RDMA overlay below. +# - NCCL_IB_GID_INDEX=3 : RoCEv2 IPv4 GID index. Check `show_gids`; v1/v2 and +# IPv4/IPv6 live at different indices per port. +# - NCCL_IB_TC=104 : RoCE lossless (PFC) traffic class. Fabric/switch-specific. +# - RDMA overlay (provision phase): only needed when the CONTAINER's rdma-core +# is older than the HOST kernel driver's uapi (our bnxt_re v34-vs-v59 case). +# Different NIC/host => different /usr/lib64 provider .so to stage, or the +# overlay may be unnecessary entirely (set RDMA_OVERLAY= to disable). If RDMA +# can't be made to work, NCCL_NET_TRANSPORT=socket falls back to TCP. +# +# E) Not cluster-specific (auto-derived): GPUS_PER_NODE (torch.cuda.device_count), +# NNODES/NODE_RANK/MASTER_ADDR (from SLURM), WORLD_SIZE. +# ============================================================================= +set -uo pipefail + +# Absolute path to THIS script so the orchestrate phase can re-invoke it on every +# node (home is shared NFS, so the same path resolves cluster-wide). +SELF=$(cd "$(dirname "$0")" && pwd)/$(basename "$0") +REPO_ROOT=$(cd "$(dirname "$0")/.." && pwd) + +# ---- phase detection -------------------------------------------------------- +PHASE="${LAUNCH_SLURM_PHASE:-}" +if [ -z "$PHASE" ]; then + if [ -f /.dockerenv ]; then PHASE=worker; else PHASE=orchestrate; fi +fi + +# ---- shared config (env-overridable) ---------------------------------------- +CONTAINER=${CONTAINER:-yambda_primus} +REPO=${REPO:-$REPO_ROOT} # repo path inside the container +IMAGE=${IMAGE:-rocm/primus:v26.3} # [CLUSTER-SPECIFIC] ROCm/arch base image +BAKED_IMAGE=${BAKED_IMAGE:-yambda_primus_baked:latest} +BAKED_TAR=${BAKED_TAR:-/apps/chcai/yambda_primus_baked.tar} # [CLUSTER-SPECIFIC] shared-NFS path +USE_BAKED=${USE_BAKED:-1} +OVERLAY=${RDMA_OVERLAY:-/apps/chcai/rdma_host_el9_new} # [CLUSTER-SPECIFIC] shared-NFS RDMA overlay + +REPO_MOUNT=${REPO_MOUNT:-$HOME} # NFS home holding the repo (must contain $REPO); override if your repo lives elsewhere +DATA_MOUNT=${DATA_MOUNT:-/apps/chcai} # shared dataset + RDMA overlay + pip/fbgemm assets (read-only) +SCRATCH=${SCRATCH:-$HOME/yambda_runs} # writable output root (logs / tb / traces) + +# ============================================================================= +# PHASE: orchestrate (SLURM batch host) +# ============================================================================= +orchestrate() { + # When run as the SLURM batch script, $0 is the node-local staged copy + # (/var/spool/slurmd/job/slurm_script), so $SELF / $REPO_ROOT are WRONG + # here (they don't exist on other nodes). Resolve the REAL shared-NFS script + # path + repo root from SLURM so we can re-invoke this script on every node and + # `cd` to the right repo inside the container. + SCRIPT_PATH=$(scontrol show job "${SLURM_JOB_ID:-0}" 2>/dev/null | grep -oP 'Command=\K\S+') + [ -f "${SCRIPT_PATH:-}" ] || SCRIPT_PATH="${SLURM_SUBMIT_DIR:-$REPO_ROOT}/scripts/launch_slurm.sh" + [ -f "$SCRIPT_PATH" ] || SCRIPT_PATH="$SELF" + REPO=$(cd "$(dirname "$SCRIPT_PATH")/.." && pwd) + + mkdir -p "$SCRATCH" 2>/dev/null || true + LOG=${LOG:-$SCRATCH/yambda_slurm.${SLURM_JOB_ID:-manual}.log} + + # Smoke defaults — override via env for a perf run (see header USAGE). + MODE=${MODE:-streaming-train-eval} + START_TS=${START_TS:-150} + NUM_TRAIN_TS=${NUM_TRAIN_TS:-1} + NUM_TRAIN_BATCHES=${NUM_TRAIN_BATCHES:-20} + NUM_EVAL_BATCHES=${NUM_EVAL_BATCHES:-10} + EVAL_EACH_WINDOW=${EVAL_EACH_WINDOW:-1} + # Default eval cadence: per-window OFF (0), data-fraction every 0.5% of data + # (0.005). The two are mutually exclusive (both >0 raises a ValueError). + EVAL_EVERY_N_WINDOWS=${EVAL_EVERY_N_WINDOWS:-0} + EVAL_EVERY_DATA_PCT=${EVAL_EVERY_DATA_PCT:-0.005} + METRIC_LOG_FREQ=${METRIC_LOG_FREQ:-5} + FORCE_PROVISION=${FORCE_PROVISION:-0} + + # Truncate the metrics log on a FRESH run; APPEND on a supervised relaunch + # (APPEND_LOG=1) so the full-run NE/AUC history survives crash/node-failover + # resubmits instead of being wiped on every attempt (mirrors the single-node + # supervisor's init-once/append model). + if [ "${APPEND_LOG:-0}" = "1" ]; then + echo "[$(date)] === resume: appending to existing $LOG (APPEND_LOG=1) ===" >> "$LOG" + else + : > "$LOG" + fi + echo "[$(date)] launch_slurm/orchestrate: job=${SLURM_JOB_ID:-?} nodes=${SLURM_JOB_NODELIST:-?} nnodes=${SLURM_NNODES:-1}" | tee -a "$LOG" + echo "[$(date)] resolved SCRIPT_PATH=$SCRIPT_PATH REPO=$REPO" | tee -a "$LOG" + echo "[$(date)] config: MODE=$MODE START_TS=$START_TS NUM_TRAIN_TS=$NUM_TRAIN_TS NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES METRIC_LOG_FREQ=$METRIC_LOG_FREQ" | tee -a "$LOG" + + # Rendezvous resolved on the HOST (the container image has no SLURM client). + MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) + MASTER_ADDR=${MASTER_ADDR:-localhost} + MASTER_PORT=$(( 20000 + ${SLURM_JOB_ID:-0} % 20000 )) + echo "[$(date)] rendezvous: MASTER_ADDR=$MASTER_ADDR MASTER_PORT=$MASTER_PORT" | tee -a "$LOG" + + # Optional NCCL/RCCL fabric overrides — forwarded into the container only when + # set at submit time (docker exec does NOT inherit the srun task env). The + # worker phase applies its own validated multi-node bnxt_re defaults when these + # are unset. Common: NCCL_NET_TRANSPORT=socket (TCP fallback), NCCL_DEBUG=INFO. + NCCL_ENV_ARGS="" + for v in NCCL_NET_TRANSPORT NCCL_DEBUG NCCL_SOCKET_IFNAME NCCL_IB_HCA NCCL_IB_GID_INDEX \ + NCCL_IB_TC NCCL_IB_TIMEOUT NCCL_IGNORE_CPU_AFFINITY RCCL_MSCCL_ENABLE NCCL_NET_GDR_LEVEL \ + NCCL_IB_PCI_RELAXED_ORDERING NCCL_IB_USE_INLINE NCCL_IB_QPS_PER_CONNECTION \ + NCCL_IB_ECE_ENABLE NCCL_DMABUF_ENABLE NCCL_GDRCOPY_ENABLE NCCL_GDR_FLUSH_DISABLE \ + NCCL_PXN_DISABLE NCCL_CHECKS_DISABLE NCCL_CROSS_NIC RDMA_OVERLAY; do + eval "val=\${$v:-}" + if [ -n "$val" ]; then NCCL_ENV_ARGS="$NCCL_ENV_ARGS -e $v=$val"; fi + done + + # TRICKY — variable expansion inside the `srun ... bash -c "..."` blocks below: + # the string is double-quoted, so PLAIN $VAR expands NOW on the batch host (e.g. + # $MASTER_ADDR, $CONTAINER, $SCRIPT_PATH — values computed above), while + # BACKSLASH-escaped \$VAR is passed through literally and expands LATER on each + # compute node inside the srun task (e.g. \$SLURM_NODEID, \$(hostname)) where the + # per-node SLURM_* env actually lives. Mixing these up sends every rank the + # wrong node id or breaks the docker exec — keep the \$ on per-node values. + + # --- step 1: ensure the container is up on every node ---------------------- + echo "[$(date)] ensuring container '$CONTAINER' on all nodes (force=$FORCE_PROVISION)" | tee -a "$LOG" + srun --ntasks-per-node=1 bash -c " + # Reap stale/foreign GPU containers from prior jobs BEFORE (re)provisioning. + # The node is allocated --exclusive, so any GPU container other than + # '$CONTAINER' is an orphan left by a previous job (its container outlives the + # SLURM allocation). We remove every such container that has GPU access + # (/dev/kfd or /dev/dri) — running OR stopped, whether or not it currently + # pins VRAM ('docker ps -aq' includes stopped ones) — since idle orphans can + # still hold device handles or wake up; leaked HBM from these has caused both + # OOMs and RCCL collective hangs. We deliberately SKIP non-GPU containers + # (e.g. 'k8s-node-services-*' and other cluster system services) so we don't + # disrupt node infrastructure. docker teardown lets the driver reclaim HBM. + for _c in \$(docker ps -aq 2>/dev/null); do + _nm=\$(docker inspect -f '{{.Name}}' \"\$_c\" 2>/dev/null | sed 's#^/##') + [ \"\$_nm\" = \"$CONTAINER\" ] && continue + _dev=\$(docker inspect -f '{{range .HostConfig.Devices}}{{.PathOnHost}} {{end}}' \"\$_c\" 2>/dev/null) + case \"\$_dev\" in + *kfd*|*dri*) + echo \"[\$(hostname)] reaping stale GPU container \$_nm (\$_c)\" + docker rm -f \"\$_c\" >/dev/null 2>&1 || true ;; + *) + echo \"[\$(hostname)] keeping non-GPU/system container \$_nm (\$_c)\" ;; + esac + done + # Reuse a STOPPED '$CONTAINER' (its installed deps persist in the container + # fs) instead of destructively re-provisioning from the base image + pip. + # Harmless no-op on a fresh node (no such container) -> falls through to + # provision below. Repo code is bind-mounted, so live edits are still picked up. + docker start $CONTAINER >/dev/null 2>&1 || true + if [ \"$FORCE_PROVISION\" = \"1\" ] || ! docker exec $CONTAINER true >/dev/null 2>&1; then + echo \"[\$(hostname)] (re)provisioning container\" + LAUNCH_SLURM_PHASE=provision CONTAINER=$CONTAINER IMAGE=$IMAGE \ + BAKED_IMAGE=$BAKED_IMAGE BAKED_TAR=$BAKED_TAR USE_BAKED=$USE_BAKED \ + BAKE_IMAGE=${BAKE_IMAGE:-0} RDMA_OVERLAY=$OVERLAY REPO=$REPO \ + REPO_MOUNT=$REPO_MOUNT DATA_MOUNT=$DATA_MOUNT SCRATCH=$SCRATCH bash $SCRIPT_PATH + else + # Container persists across jobs; the reap above only removes FOREIGN GPU + # containers, so our own '$CONTAINER' can still pin HBM via stray trainer + # ranks left by a prior OOM/crash (this caused repeated 'CUDA out of memory' + # on relaunch onto the same node). Restart it to kill every exec'd proc and + # let the driver reclaim HBM — cheap (keeps the installed deps in the + # container fs; NFS RDMA overlay also persists), no full re-provision. + echo \"[\$(hostname)] container already up — restarting to free any leaked HBM before launch\" + docker restart $CONTAINER >/dev/null 2>&1 || true + # Readiness gate: a bare 'docker exec true' can pass while the runtime is + # still settling, so the SUBSEQUENT (heavier) worker exec races the restart + # and dies with 'container is not running' / OCI 'setns' errors (observed on + # c07-08 and e08-08 -> the peer never joins rendezvous -> master 600s + # TCPStore timeout). Require State.Running=true AND a successful probe, then + # a short settle, before considering the container ready. + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + sleep 2 + done + sleep 2 + echo \"[\$(hostname)] container restarted (HBM reclaimed; running=\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null))\" + fi + " 2>&1 | tee -a "$LOG" + + # --- step 2: launch the worker (trainer) inside the container on every node - + echo "[$(date)] launching trainer (worker phase) on all nodes" | tee -a "$LOG" + srun --ntasks-per-node=1 bash -c " + # Pre-flight readiness gate (per node): step 1 ran in a SEPARATE srun, so the + # container can still be settling here. Wait for State.Running=true + a probe + # before the worker exec so we don't race a just-restarted container. + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + [ \$_w -eq 1 ] && echo \"[\$(hostname)] worker pre-flight: waiting for container to be ready...\" + sleep 2 + done + # Retry wrapper: docker exec startup failures (rc 125 daemon 'container is not + # running', 126/127 OCI/setns 'exec failed') mean the container wasn't ready, + # NOT that the trainer ran and failed. Restart + re-gate + retry a few times. + # Any OTHER rc (the trainer actually started and exited) is propagated so the + # supervisor's resume-from-checkpoint logic owns real failures. + _wattempt=0 + while : ; do + _wattempt=\$((_wattempt+1)) + docker exec \ + -e LAUNCH_SLURM_PHASE=worker \ + -e SCRATCH=$SCRATCH \ + -e SLURM_NNODES=\$SLURM_NNODES \ + -e SLURM_NODEID=\$SLURM_NODEID \ + -e SLURM_PROCID=\$SLURM_PROCID \ + -e SLURM_JOB_NODELIST=\"\$SLURM_JOB_NODELIST\" \ + -e SLURM_JOB_ID=\$SLURM_JOB_ID \ + -e MASTER_ADDR=$MASTER_ADDR \ + -e MASTER_PORT=$MASTER_PORT \ + -e HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-TRITON} \ + -e MODE=$MODE \ + -e START_TS=$START_TS \ + -e NUM_TRAIN_TS=$NUM_TRAIN_TS \ + -e EVAL_EVERY_N_WINDOWS=$EVAL_EVERY_N_WINDOWS \ + ${EVAL_EVERY_DATA_PCT:+-e EVAL_EVERY_DATA_PCT=$EVAL_EVERY_DATA_PCT} \ + -e NUM_TRAIN_BATCHES=$NUM_TRAIN_BATCHES \ + -e NUM_EVAL_BATCHES=$NUM_EVAL_BATCHES \ + ${DIE_AT_STEP:+-e DIE_AT_STEP=$DIE_AT_STEP} \ + -e METRIC_LOG_FREQ=$METRIC_LOG_FREQ \ + ${EVAL_INTERIM_LOG:+-e EVAL_INTERIM_LOG=$EVAL_INTERIM_LOG} \ + ${SKIP_EVAL:+-e SKIP_EVAL=$SKIP_EVAL} \ + ${MLPERF_LOGGING:+-e MLPERF_LOGGING=$MLPERF_LOGGING} \ + ${MLPERF_TRAIN_LOSS_LOG_FREQ:+-e MLPERF_TRAIN_LOSS_LOG_FREQ=$MLPERF_TRAIN_LOSS_LOG_FREQ} \ + ${STREAMING_SHUFFLE_FRACTION:+-e STREAMING_SHUFFLE_FRACTION=$STREAMING_SHUFFLE_FRACTION} \ + ${STREAMING_SHUFFLE_SEED:+-e STREAMING_SHUFFLE_SEED=$STREAMING_SHUFFLE_SEED} \ + ${NUM_WORKERS:+-e NUM_WORKERS=$NUM_WORKERS} \ + ${PREFETCH_FACTOR:+-e PREFETCH_FACTOR=$PREFETCH_FACTOR} \ + ${DIAG_UNIQUE_EMB:+-e DIAG_UNIQUE_EMB=$DIAG_UNIQUE_EMB} \ + ${DIAG_EMB_STEPS:+-e DIAG_EMB_STEPS=$DIAG_EMB_STEPS} \ + ${OUTPUT_TRACE:+-e OUTPUT_TRACE=$OUTPUT_TRACE} \ + ${MIN_HISTORY:+-e MIN_HISTORY=$MIN_HISTORY} \ + ${SEED:+-e SEED=$SEED} \ + ${DENSE_LR:+-e DENSE_LR=$DENSE_LR} \ + ${SPARSE_LR:+-e SPARSE_LR=$SPARSE_LR} \ + ${GRAD_CLIP_NORM:+-e GRAD_CLIP_NORM=$GRAD_CLIP_NORM} \ + ${LR_WARMUP_STEPS:+-e LR_WARMUP_STEPS=$LR_WARMUP_STEPS} \ + ${LR_WARMUP_START_LR:+-e LR_WARMUP_START_LR=$LR_WARMUP_START_LR} \ + ${HSTU_NUM_LAYERS:+-e HSTU_NUM_LAYERS=$HSTU_NUM_LAYERS} \ + ${MAX_SEQ_LEN:+-e MAX_SEQ_LEN=$MAX_SEQ_LEN} \ + ${HISTORY_LENGTH:+-e HISTORY_LENGTH=$HISTORY_LENGTH} \ + ${BATCH_SIZE:+-e BATCH_SIZE=$BATCH_SIZE} \ + ${CKPT_TIME_INTERVAL_S:+-e CKPT_TIME_INTERVAL_S=$CKPT_TIME_INTERVAL_S} \ + ${KEEP_LAST_N:+-e KEEP_LAST_N=$KEEP_LAST_N} \ + ${IN_WINDOW_CKPT_FREQ:+-e IN_WINDOW_CKPT_FREQ=$IN_WINDOW_CKPT_FREQ} \ + ${CKPT_STEP_FREQ:+-e CKPT_STEP_FREQ=$CKPT_STEP_FREQ} \ + -e TRAIN_SPLIT_PERCENTAGE=${TRAIN_SPLIT_PERCENTAGE:-1.0} \ + -e AUC_THRESHOLD=${AUC_THRESHOLD:-1.0} \ + ${MLPERF_SUBMISSION_PLATFORM:+-e MLPERF_SUBMISSION_PLATFORM=$MLPERF_SUBMISSION_PLATFORM} \ + -e SPLIT_SALT=${SPLIT_SALT:-0} \ + -e EVAL_HOLDOUT_TS=${EVAL_HOLDOUT_TS:--1} \ + -e EVAL_HOLDOUT_NUM_WINDOWS=${EVAL_HOLDOUT_NUM_WINDOWS:-1} \ + ${WORKER_CMD:+-e WORKER_CMD=\"$WORKER_CMD\"} \ + ${RUN_NAME:+-e RUN_NAME=$RUN_NAME} \ + ${TENSORBOARD_LOG_PATH:+-e TENSORBOARD_LOG_PATH=$TENSORBOARD_LOG_PATH} \ + ${MLPERF_LOG_PATH:+-e MLPERF_LOG_PATH=$MLPERF_LOG_PATH} \ + ${CKPT_PATH:+-e CKPT_PATH=$CKPT_PATH} \ + ${SPARSE_A2A_FWD:+-e SPARSE_A2A_FWD=$SPARSE_A2A_FWD} \ + ${SPARSE_A2A_BWD:+-e SPARSE_A2A_BWD=$SPARSE_A2A_BWD} \ + ${QCOMM_LOWMEM_CODEC:+-e QCOMM_LOWMEM_CODEC=$QCOMM_LOWMEM_CODEC} \ + ${EC_INDEX_DEDUP:+-e EC_INDEX_DEDUP=$EC_INDEX_DEDUP} \ + ${EMB_SHARDING_OVERRIDES:+-e EMB_SHARDING_OVERRIDES=$EMB_SHARDING_OVERRIDES} \ + ${EMB_PLACEMENT_OVERRIDES:+-e EMB_PLACEMENT_OVERRIDES=$EMB_PLACEMENT_OVERRIDES} \ + ${EMB_PLACEMENT:+-e EMB_PLACEMENT=$EMB_PLACEMENT} \ + ${PG_TIMEOUT_S:+-e PG_TIMEOUT_S=$PG_TIMEOUT_S} \ + ${TORCH_NCCL_TRACE_BUFFER_SIZE:+-e TORCH_NCCL_TRACE_BUFFER_SIZE=$TORCH_NCCL_TRACE_BUFFER_SIZE} \ + ${TORCH_NCCL_DUMP_ON_TIMEOUT:+-e TORCH_NCCL_DUMP_ON_TIMEOUT=$TORCH_NCCL_DUMP_ON_TIMEOUT} \ + ${TORCH_NCCL_TRACE_CPP_STACK:+-e TORCH_NCCL_TRACE_CPP_STACK=$TORCH_NCCL_TRACE_CPP_STACK} \ + ${TORCH_NCCL_DEBUG_INFO_TEMP_FILE:+-e TORCH_NCCL_DEBUG_INFO_TEMP_FILE=$TORCH_NCCL_DEBUG_INFO_TEMP_FILE} \ + -e LOG=$LOG \ + $NCCL_ENV_ARGS \ + $CONTAINER bash -lc 'cd $REPO && LAUNCH_SLURM_PHASE=worker bash scripts/launch_slurm.sh' + _wrc=\$? + if { [ \$_wrc -eq 125 ] || [ \$_wrc -eq 126 ] || [ \$_wrc -eq 127 ]; } && [ \$_wattempt -lt 5 ]; then + echo \"[\$(hostname)] worker exec failed to START (rc=\$_wrc, attempt \$_wattempt/5) — container not ready; restarting + retrying\" + docker restart $CONTAINER >/dev/null 2>&1 || true + for _w in \$(seq 1 30); do + [ \"\$(docker inspect -f '{{.State.Running}}' $CONTAINER 2>/dev/null)\" = \"true\" ] \ + && docker exec $CONTAINER true >/dev/null 2>&1 && break + sleep 2 + done + sleep 3 + continue + fi + exit \$_wrc + done + " 2>&1 | tee -a "$LOG" + rc=${PIPESTATUS[0]} + echo "[$(date)] launch_slurm/orchestrate finished rc=$rc" | tee -a "$LOG" + exit $rc +} + +# ============================================================================= +# PHASE: provision (compute-node host) +# ============================================================================= +provision() { + export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:${PATH:-}" + DOCKER=$(command -v docker 2>/dev/null || true); DOCKER=${DOCKER:-/usr/bin/docker} + FBGEMM_WHL=${FBGEMM_WHL:-/apps/chcai/FBGEMM/fbgemm_gpu/dist/fbgemm_gpu_nightly_rocm-2026.6.2-cp312-cp312-linux_x86_64.whl} # [CLUSTER-SPECIFIC] gfx950/ROCm wheel + TORCH_IDX=${TORCH_IDX:-https://download.pytorch.org/whl/rocm7.2} # [CLUSTER-SPECIFIC] ROCm version index + echo "[provision] host=$(hostname) container=$CONTAINER docker=$DOCKER" + + # Resolve which image to run + whether deps must be installed. Prefer a pre-baked + # image (deps already installed) to skip the multi-GB torch download + pip / + # torchrec-from-git build on every fresh node: + # 1) baked image in this node's docker -> use it, skip deps + # 2) baked image tar on NFS -> docker load (local, no internet) + # 3) neither -> base image + pip (slow path, which + # can then be baked via BAKE_IMAGE=1) + NEED_DEPS=1 + RUN_IMAGE="$IMAGE" + if [ "$USE_BAKED" = "1" ]; then + if "$DOCKER" image inspect "$BAKED_IMAGE" >/dev/null 2>&1; then + echo "[provision] using baked image $BAKED_IMAGE (deps preinstalled, no download)" + RUN_IMAGE="$BAKED_IMAGE"; NEED_DEPS=0 + elif [ -f "$BAKED_TAR" ]; then + echo "[provision] loading baked image from $BAKED_TAR (local, no internet)..." + if "$DOCKER" load -i "$BAKED_TAR" >/dev/null 2>&1 && "$DOCKER" image inspect "$BAKED_IMAGE" >/dev/null 2>&1; then + RUN_IMAGE="$BAKED_IMAGE"; NEED_DEPS=0; echo "[provision] baked image loaded" + else + echo "[provision] WARNING: docker load failed; falling back to base-image + pip" + fi + fi + fi + if ! "$DOCKER" image inspect "$RUN_IMAGE" >/dev/null 2>&1; then + echo "[provision] pulling $RUN_IMAGE (this can take a while)..."; "$DOCKER" pull "$RUN_IMAGE" + fi + + echo "[provision] (re)starting container $CONTAINER from $RUN_IMAGE" + "$DOCKER" rm -f "$CONTAINER" >/dev/null 2>&1 || true + "$DOCKER" run -d --name "$CONTAINER" \ + --network=host --ipc=host --shm-size=64g \ + --device=/dev/kfd --device=/dev/dri --group-add video \ + `# [CLUSTER-SPECIFIC] AMD ROCm device passthrough; NVIDIA uses --gpus all / nvidia runtime` \ + --cap-add=SYS_PTRACE --cap-add=CAP_SYS_ADMIN --cap-add=IPC_LOCK \ + --ulimit memlock=-1:-1 --ulimit stack=67108864:67108864 \ + `# memlock=-1 is REQUIRED for RDMA QP memory registration — do not drop` \ + --security-opt seccomp=unconfined --privileged \ + -v "$REPO_MOUNT:$REPO_MOUNT" \ + -v "$DATA_MOUNT:$DATA_MOUNT" \ + `# shared-NFS bind mounts: repo home (REPO_MOUNT, rw) + dataset/build assets (DATA_MOUNT)` \ + -w "$REPO" \ + "$RUN_IMAGE" sleep infinity + + # --- RDMA userspace overlay for in-container RCCL (bnxt_re) ----------------- + # The image (rocm/primus, rdma-core 50/libbnxt_re-rdmav34) ships an OLDER RDMA + # userspace than the host kernel bnxt_re driver. The stock v34 provider faults + # RCCL's deep-queue create_qp (max_send_wr=256) against the newer kernel uapi + # -> "ibv_create_qp ... Bad address". Fix: stage the host's matched rdma-core + # (libibverbs v61 + libbnxt_re-rdmav59 + libnl) on NFS so the worker phase makes + # RCCL load it via LD_PRELOAD + LD_LIBRARY_PATH. The UNVERSIONED libibverbs.so + # symlink is essential (import torch pulls the unversioned soname; without it + # the lookup falls through to the container v34 lib and the fix regresses). + if [ "${FORCE_OVERLAY:-0}" != "1" ] && ls "$OVERLAY/lib/libibverbs/"libbnxt_re-rdmav*.so >/dev/null 2>&1 && [ -L "$OVERLAY/lib/libibverbs.so" ]; then + echo "[provision] host RDMA overlay already staged at $OVERLAY (shared NFS) — skipping" + else + echo "[provision] staging host RDMA userspace overlay -> $OVERLAY" + rm -rf "${OVERLAY}.tmp" 2>/dev/null + mkdir -p "${OVERLAY}.tmp/lib/libibverbs" "${OVERLAY}.tmp/libibverbs.d" + cp -L /usr/lib64/libibverbs.so.1 /usr/lib64/libnl-3.so.200 /usr/lib64/libnl-route-3.so.200 "${OVERLAY}.tmp/lib/" 2>/dev/null || true + ln -sf libibverbs.so.1 "${OVERLAY}.tmp/lib/libibverbs.so" + cp -L /usr/lib64/libibverbs/*.so "${OVERLAY}.tmp/lib/libibverbs/" 2>/dev/null || true + cp /etc/libibverbs.d/*.driver "${OVERLAY}.tmp/libibverbs.d/" 2>/dev/null || true + if ls "${OVERLAY}.tmp/lib/libibverbs/"libbnxt_re-rdmav*.so >/dev/null 2>&1; then + rm -rf "$OVERLAY" 2>/dev/null + mv "${OVERLAY}.tmp" "$OVERLAY" 2>/dev/null || { mkdir -p "$OVERLAY"; cp -a "${OVERLAY}.tmp/." "$OVERLAY/"; } + echo "[provision] host RDMA overlay staged: $(ls "$OVERLAY/lib/libibverbs" | wc -l) providers + libibverbs.so symlink" + else + echo "[provision] WARNING: host bnxt_re provider not found at /usr/lib64/libibverbs — multi-node RDMA will fail 'Bad address'; use NCCL_NET_TRANSPORT=socket" + fi + fi + + if [ "$NEED_DEPS" = "0" ]; then + echo "[provision] baked image — deps preinstalled; verifying imports only" + "$DOCKER" exec "$CONTAINER" bash -lc ' +python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"imports OK,\", torch.__version__, torch.version.hip, torch.cuda.device_count(), \"gpus\")" +' || echo "[provision] WARNING: baked-image import smoke failed" + else + echo "[provision] installing recipe deps (base image, slow path)" + # Install misc deps FIRST, then pin the rocm torch stack + fbgemm + torchrec + # LAST with --no-deps so nothing pulls a CUDA torch over the rocm build. + "$DOCKER" exec "$CONTAINER" bash -lc ' +set -e +echo "=== native torch ==="; python -c "import torch;print(torch.__version__)" || true +echo "=== misc python deps ===" +pip install --no-cache-dir polars-u64-idx pyarrow pyyaml tqdm psutil numba xxhash gin-config \ + absl-py pandas tensorboard torchmetrics tensordict pyre-extensions iopath typing-inspect 2>&1 | tail -3 || true +echo "=== rocm torch stack (force, no-deps, LAST) ===" +pip install --force-reinstall --no-deps --index-url '"$TORCH_IDX"' \ + torch==2.12.0+rocm7.2 torchvision==0.27.0+rocm7.2 torchaudio==2.11.0+rocm7.2 +echo "=== fbgemm (local gfx950 wheel) ===" +pip install --force-reinstall --no-deps '"$FBGEMM_WHL"' +echo "=== torchrec v2026.06.01.00 (force, no-deps) ===" +pip install --force-reinstall --no-deps "git+https://github.com/pytorch/torchrec.git@v2026.06.01.00" +echo "=== import smoke ===" +python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print(\"imports OK,\", torch.__version__, torch.version.hip, torch.cuda.device_count(), \"gpus\")" +' + fi + + # --- one-time bake: snapshot the deps-installed container into a reusable image + # and save it to NFS so future nodes skip the download/pip path entirely. + if [ "${BAKE_IMAGE:-0}" = "1" ]; then + echo "[provision] baking: docker commit $CONTAINER -> $BAKED_IMAGE" + if "$DOCKER" commit "$CONTAINER" "$BAKED_IMAGE" >/dev/null; then + echo "[provision] saving $BAKED_IMAGE -> $BAKED_TAR (one-time, tens of GB)" + if "$DOCKER" save "$BAKED_IMAGE" -o "${BAKED_TAR}.tmp.$$" && mv -f "${BAKED_TAR}.tmp.$$" "$BAKED_TAR"; then + echo "[provision] bake done: $(ls -lh "$BAKED_TAR" 2>/dev/null | awk '{print $5}')" + else + echo "[provision] WARNING: docker save failed"; rm -f "${BAKED_TAR}.tmp.$$" 2>/dev/null + fi + else + echo "[provision] WARNING: docker commit failed" + fi + fi + echo "[provision] DONE" +} + +# ============================================================================= +# PHASE: worker (inside the container) +# ============================================================================= +worker() { + cd "$REPO_ROOT" + mkdir -p "$SCRATCH" 2>/dev/null || true + LOG=${LOG:-$SCRATCH/yambda_5b_8gpu.log} + # Append (not truncate): under the streaming-e2e supervisor a run may relaunch + # many times into the SAME $LOG; the supervisor initializes it once at run start. + # MLPerf compliance log (rank 0 writes it). Per-job filename so each standalone + # sbatch gets a clean log; the e2e supervisor pins MLPERF_LOG_PATH itself. + export MLPERF_LOG_PATH=${MLPERF_LOG_PATH:-$SCRATCH/mlperf/yambda_5b_mlperf.${SLURM_JOB_ID:-manual}.log} + echo "[$(date)] REPO_ROOT=$REPO_ROOT" | tee -a "$LOG" + + # polars-u64-idx (NOT stock polars) — yambda parquet's flat-explode overruns + # 32-bit row index. Reserved node has no outbound DNS, so install from a + # pre-staged tarball under /apps/chcai/. Override PIP_LOCAL_TGZ for other hosts. + PIP_LOCAL_TGZ=${PIP_LOCAL_TGZ:-/apps/chcai/pip_local_yambda.tgz} # [CLUSTER-SPECIFIC] shared-NFS path + PIP_LOCAL_DIR=${PIP_LOCAL_DIR:-/tmp/pip_local} + if [ ! -f "$PIP_LOCAL_DIR/lib/python3.12/site-packages/polars/__init__.py" ]; then + rm -rf "$PIP_LOCAL_DIR" + mkdir -p "$PIP_LOCAL_DIR" && tar xzf "$PIP_LOCAL_TGZ" -C "$(dirname "$PIP_LOCAL_DIR")" 2>&1 | tail -3 | tee -a "$LOG" + fi + + export PYTHONPATH="$PIP_LOCAL_DIR/lib/python3.12/site-packages:$REPO_ROOT:${PYTHONPATH:-}" + export HOME=${HOME:-/tmp} + echo "[$(date)] PYTHONPATH=$PYTHONPATH" | tee -a "$LOG" + python -c "import torch, fbgemm_gpu, torchrec, polars, xxhash, gin; print('imports OK,', torch.__version__, torch.cuda.device_count(),'gpus')" 2>&1 | tee -a "$LOG" + + export HIP_VISIBLE_DEVICES=${HIP_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} + export CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-0,1,2,3,4,5,6,7} + + # --- distributed topology --------------------------------------------------- + GPUS_PER_NODE=$(python -c "import torch; print(torch.cuda.device_count())") + # Multi-node when launched one-task-per-node under SLURM (SLURM_NNODES>1); + # otherwise fall through to legacy single-node defaults (localhost, node_rank 0). + if [ "${SLURM_NNODES:-1}" -gt 1 ] && [ -n "${SLURM_JOB_NODELIST:-}" ]; then + NNODES=${SLURM_NNODES} + NODE_RANK=${SLURM_NODEID:-${SLURM_PROCID:-0}} + # PREFER a MASTER_ADDR/PORT forwarded from the orchestrate phase (resolved on + # the host, which has scontrol); the container image carries no SLURM client. + if [ -z "${MASTER_ADDR:-}" ]; then + MASTER_ADDR=$(scontrol show hostnames "$SLURM_JOB_NODELIST" 2>/dev/null | head -1) + MASTER_ADDR=${MASTER_ADDR:-localhost} + fi + MASTER_PORT=${MASTER_PORT:-$(( 20000 + ${SLURM_JOB_ID:-0} % 20000 ))} + else + NNODES=${NNODES:-1} + NODE_RANK=${NODE_RANK:-0} + # Single-node: all ranks live on THIS host, so rendezvous over loopback and + # do NOT use the SLURM hostname. On some nodes the hostname resolves to a + # non-routable per-GPU RoCE /31 (benic 192.168.x) address; using it makes the + # NCCL bootstrap fail with "No route to host". localhost is node-independent. + MASTER_ADDR=localhost + MASTER_PORT=${MASTER_PORT:-} # empty => train_ranker picks a free port + fi + export NNODES NODE_RANK GPUS_PER_NODE MASTER_ADDR MASTER_PORT + export WORLD_SIZE=$(( NNODES * GPUS_PER_NODE )) + echo "[$(date)] topology: nnodes=$NNODES node_rank=$NODE_RANK gpus_per_node=$GPUS_PER_NODE world_size=$WORLD_SIZE master=$MASTER_ADDR:${MASTER_PORT:-}" | tee -a "$LOG" + + # NCCL bootstrap NIC — pin for BOTH single- and multi-node. The container is + # --network=host so RCCL sees ALL host interfaces; if left to auto-detect, NCCL + # can pick a non-routable per-GPU RoCE /31 (benic* 192.168.x) link and fail + # bootstrap with "No route to host" (this is node-dependent: it happened to + # work on some nodes and not others, causing repetitive single-node init + # failures). Pinning the routable host NIC fixes it everywhere. + # [CLUSTER-SPECIFIC] routable host NIC for TCP bootstrap (find via `ip -br addr`). + export NCCL_SOCKET_IFNAME=${NCCL_SOCKET_IFNAME:-fenic0} + + # Multi-node additionally needs the RDMA data-plane (bnxt_re HCAs) configured; + # single-node uses intra-node P2P (XGMI/PCIe) so only the bootstrap NIC matters. + if [ "$NNODES" -gt 1 ]; then + NCCL_NET_TRANSPORT=${NCCL_NET_TRANSPORT:-ib} + if [ "$NCCL_NET_TRANSPORT" = "socket" ]; then + export NCCL_IB_DISABLE=1 + echo "[$(date)] NCCL: IB disabled — allreduce over TCP (fenic0). Functional, not RDMA-fast." | tee -a "$LOG" + else + # bnxt_re userspace provider ABI overlay (REQUIRED for RCCL). The stock v34 + # provider faults RCCL's create_qp (256 WRs) against the host kernel uapi + # ("Bad address"); the host v61/v59 set staged by the provision phase works. + # The libibverbs.so (UNVERSIONED) symlink + LD_PRELOAD are both required so + # the torch process maps ONLY the host lib (see provision phase comment). + if [ -e "$OVERLAY/lib/libibverbs.so.1" ]; then + [ -e "$OVERLAY/lib/libibverbs.so" ] || ln -sf libibverbs.so.1 "$OVERLAY/lib/libibverbs.so" 2>/dev/null || true + export LD_LIBRARY_PATH="$OVERLAY/lib:$OVERLAY/lib/libibverbs:${LD_LIBRARY_PATH:-}" + export LD_PRELOAD="$OVERLAY/lib/libibverbs.so.1${LD_PRELOAD:+:$LD_PRELOAD}" + echo "[$(date)] NCCL: bnxt_re provider overlay -> $OVERLAY (host rdma-core v61/v59; symlink+LD_PRELOAD so RCCL binds the host lib for QP creation)" | tee -a "$LOG" + else + echo "[$(date)] WARNING: RDMA overlay $OVERLAY missing — RCCL QP creation will fail 'Bad address' on stock v34 provider; set RDMA_OVERLAY or use NCCL_NET_TRANSPORT=socket" | tee -a "$LOG" + fi + # MINIMAL bnxt_re set PROVEN on these meta64 cv350 nodes (cmcknigh RCCL + # benchmarks + confirmed e2e here). NCCL_IB_TC=104 (RoCE lossless PFC class) + # is required; do NOT add the ionic-AINIC QPS/ECE/DMABUF block. + # [CLUSTER-SPECIFIC] RDMA HCA names (`ibv_devices`); other vendors => mlx5_*/ionic_* + export NCCL_IB_HCA=${NCCL_IB_HCA:-bnxt_re0,bnxt_re1,bnxt_re2,bnxt_re3,bnxt_re4,bnxt_re5,bnxt_re6,bnxt_re7} + export NCCL_IB_GID_INDEX=${NCCL_IB_GID_INDEX:-3} # [CLUSTER-SPECIFIC] RoCEv2 IPv4 GID idx (`show_gids`) + export NCCL_IB_TC=${NCCL_IB_TC:-104} # [CLUSTER-SPECIFIC] RoCE lossless/PFC traffic class + export NCCL_IB_TIMEOUT=${NCCL_IB_TIMEOUT:-14} + export NCCL_IGNORE_CPU_AFFINITY=${NCCL_IGNORE_CPU_AFFINITY:-1} + export RCCL_MSCCL_ENABLE=${RCCL_MSCCL_ENABLE:-0} + # GPU-Direct RDMA: ENABLED by default. The brcmrdma host kernel ships the + # inbox peer-memory client (`ib_register_peer_memory_client` in + # /proc/kallsyms), so RCCL does true GPU<->NIC DMA over bnxt_re instead of + # bouncing through host memory. Measured ~+22% throughput at 2 nodes + # (65.7%->79.8% weak-scaling efficiency) vs the old host-staged path. + # GDR_LEVEL=5 (most permissive) is required so GDR is used even when the GPU + # and NIC cross the CPU root complex. NCCL_DMABUF_ENABLE=1 is a harmless + # no-op here (kernel lacks CONFIG_DMABUF_MOVE_NOTIFY/CONFIG_PCI_P2PDMA, so + # peermem carries it). Enabling is non-fatal: if peermem is ever absent RCCL + # just logs "GDR 0" and falls back to host staging. Override with + # NCCL_NET_GDR_LEVEL=0 to force the legacy host-staged path. + export NCCL_NET_GDR_LEVEL=${NCCL_NET_GDR_LEVEL:-5} + export NCCL_DMABUF_ENABLE=${NCCL_DMABUF_ENABLE:-1} + echo "[$(date)] NCCL: RDMA over bnxt_re (GID idx ${NCCL_IB_GID_INDEX}, TC ${NCCL_IB_TC}, GDR_LEVEL=${NCCL_NET_GDR_LEVEL}, DMABUF=${NCCL_DMABUF_ENABLE}; meta64 bnxt_re config, validated)" | tee -a "$LOG" + fi + fi + export NCCL_DEBUG=${NCCL_DEBUG:-WARN} + export HSTU_HAMMER_KERNEL=${HSTU_HAMMER_KERNEL:-} + export PYTORCH_CUDA_ALLOC_CONF=${PYTORCH_CUDA_ALLOC_CONF:-expandable_segments:True} + + # --- GPU clock sanity guard ------------------------------------------------- + # A leftover perf_determinism cap (half clock) silently slows every kernel ~1.9x. + # Log the perf level + a live sclk sample and try to restore boost (non-fatal). + if command -v rocm-smi >/dev/null 2>&1; then + echo "[$(date)] GPU perf-level check:" | tee -a "$LOG" + rocm-smi --showperflevel 2>/dev/null | grep -iE "GPU\[[0-9]+\]" | tee -a "$LOG" || true + if rocm-smi --showperflevel 2>/dev/null | grep -iqE "Performance Level: *(perf_determinism|manual|low)"; then + echo "[$(date)] WARNING: GPUs not in 'auto' perf level — attempting --setperflevel auto" | tee -a "$LOG" + rocm-smi --setperflevel auto 2>/dev/null | grep -iE "set to auto" | tee -a "$LOG" \ + || echo "[$(date)] WARNING: could not set perf level (no permission?). Run 'rocm-smi --setperflevel auto' on the HOST before benchmarking — clocks may be capped." | tee -a "$LOG" + fi + echo "[$(date)] sclk sample (GPU0):$(rocm-smi -d 0 --showclocks 2>/dev/null | grep -i 'sclk clock level' | sed -E 's/.*sclk clock level//')" | tee -a "$LOG" || true + fi + + # --- stray-trainer / leaked-VRAM guard ------------------------------------- + # The trainer runs via `docker exec` into a long-lived container, so its procs + # live in the container PID namespace, NOT the SLURM job cgroup. If a prior job + # OOM'd/crashed, a rank can leak and keep holding ~half of every GPU's VRAM, + # which persists across jobs (container survives) and guarantees the next + # attempt OOMs. Before launching, reap any pre-existing trainer procs (there + # should be none at this point) and wait for VRAM to drain. [g]-guard avoids + # self-match. Non-fatal. + if pgrep -f '[g]enerative_recommenders' >/dev/null 2>&1; then + echo "[$(date)] WARNING: leaked trainer procs found pre-launch — killing." | tee -a "$LOG" + pkill -9 -f '[g]enerative_recommenders' 2>/dev/null || true + for _i in $(seq 1 15); do + pgrep -f '[g]enerative_recommenders' >/dev/null 2>&1 || break + sleep 2 + done + sleep 5 # let the driver release VRAM after process exit + if command -v rocm-smi >/dev/null 2>&1; then + echo "[$(date)] post-cleanup GPU0 used GiB:$(rocm-smi --showmeminfo vram 2>/dev/null | awk -F: '/Used/{printf " %.0f", $3/1073741824; exit}')" | tee -a "$LOG" + fi + fi + + # WORKER_CMD override: run an arbitrary in-container command (e.g. an a2a/RCCL + # micro-benchmark) instead of the trainer, REUSING all the NCCL/RDMA/topology + # setup above so it exercises the exact transport the trainer uses. The + # supervisor never sets WORKER_CMD, so the training path is unchanged. + if [ -n "${WORKER_CMD:-}" ]; then + echo "[$(date)] WORKER_CMD override (WORLD_SIZE=$WORLD_SIZE): $WORKER_CMD" | tee -a "$LOG" + bash -lc "cd $REPO_ROOT && $WORKER_CMD" 2>&1 | tee -a "$LOG" + return + fi + + echo "[$(date)] launching train_ranker with WORLD_SIZE=$WORLD_SIZE" | tee -a "$LOG" + python -m generative_recommenders.dlrm_v3.train.train_ranker \ + --dataset yambda-5b --mode "${MODE:-streaming-train-eval}" 2>&1 | tee -a "$LOG" +} + +# ---- dispatch --------------------------------------------------------------- +case "$PHASE" in + orchestrate) orchestrate ;; + provision) provision ;; + worker) worker ;; + *) echo "launch_slurm.sh: unknown LAUNCH_SLURM_PHASE='$PHASE'" >&2; exit 2 ;; +esac diff --git a/recommendation_v4/scripts/run_streaming_e2e.sh b/recommendation_v4/scripts/run_streaming_e2e.sh new file mode 100755 index 000000000..c913bac7b --- /dev/null +++ b/recommendation_v4/scripts/run_streaming_e2e.sh @@ -0,0 +1,295 @@ +#!/bin/bash +# ============================================================================= +# run_streaming_e2e.sh — self-healing supervisor for a yambda-5b streaming +# train+eval run (sbatch-job level). Works for 1..N nodes. +# ============================================================================= +# +# WHAT IT SUPERVISES +# The run is an `sbatch [--nodes=N] scripts/launch_slurm.sh` BATCH job. That +# batch script is fully self-contained: it runs orchestrate -> provision +# (container + RDMA) -> worker (in-container trainer) on EVERY node, so it +# handles single-node (--nodes=1) and multi-node (world_size=8N) identically. +# This supervisor wraps THAT job: it monitors it and, on crash / node-failure +# / hang, RESUBMITS it (which resumes from the latest checkpoint via load +# auto-latest), bounded by --max-relaunch. There is no docker-exec lifecycle +# or in-place node failover here — node replacement is SLURM's job on resubmit. +# +# RESUME MODEL (why a resubmit "just works") +# The trainer checkpoints to $CKPT_PATH and on startup load_dmp_checkpoint +# auto-resolves to the highest-numbered subdir, restoring model+optimizer+RNG +# and skipping already-trained batches of a partial window. So resubmitting the +# SAME submit-script (same CKPT_PATH/LOG) continues from where it died. +# Resubmits set APPEND_LOG=1 so the metrics log is preserved across attempts. +# +# WHAT IT DETECTS (poll every --poll-s) +# * job left the queue -> read sacct State/ExitCode: +# COMPLETED+0 => run finished (success, exit 0) +# CANCELLED => user intent (stop, exit 0 — NOT our place to resubmit) +# FAILED/NODE_FAIL/TIMEOUT/OUT_OF_MEMORY/BOOT_FAIL/PREEMPTED => relaunch +# * hang watchdog: job RUNNING but LOG frozen >= --stall-s AND no trainer +# process alive on ANY node (cross-node pgrep) => scancel + relaunch. +# * disk guard before each (re)submit: require --min-free-gib on the ckpt vol. +# +# WHERE IT RUNS +# On the SLURM head node (NFS-mounted /home/chcai code + /apps/chcai +# ckpts/logs are visible here for squeue/sacct/df and the cross-node pgrep). +# +# USAGE +# # Submit a fresh job from the launch script, then supervise it: +# nohup bash scripts/run_streaming_e2e.sh \ +# --submit-script /apps/chcai/yambda_5b_e2e//launch_1node.sh \ +# --log /apps/chcai/yambda_5b_e2e//.log \ +# --ckpt-path /apps/chcai/yambda_5b_e2e//ckpts \ +# --run-name \ +# > /apps/chcai/yambda_5b_e2e//.supervisor.console.log 2>&1 & +# +# # Adopt an already-submitted job instead of submitting a new one: +# nohup bash scripts/run_streaming_e2e.sh --jobid 13235 \ +# --submit-script .../launch_2node.sh --log .../run.log \ +# --ckpt-path .../ckpts --run-name > .../console.log 2>&1 & +# +# The node count, partition, and reservation all live in the --submit-script's +# sbatch line (launch_1node.sh / launch_2node.sh / ...), not here. +# +# EXIT CODES +# 0 run completed (COMPLETED+0) or user-cancelled +# 1 exhausted --max-relaunch without completion (or submit failed) +# 3 disk guard tripped +# ============================================================================= +set -uo pipefail + +JOBID="" # adopt this job; empty => submit fresh +SUBMIT_SCRIPT="" +LOG="" +CKPT_PATH="" +RUN_NAME="yambda_5b_e2e" +CONTAINER=yambda_primus +MAX_RELAUNCH=50 +MIN_FREE_GIB=1200 +STALL_S=2400 # 40 min: comfortably exceeds a full-holdout eval + # window + a blocking ckpt save; only trips when + # the log is frozen AND no trainer proc is alive. +POLL_S=30 + +while [[ $# -gt 0 ]]; do + case $1 in + --jobid) JOBID="$2"; shift 2;; + --submit-script) SUBMIT_SCRIPT="$2"; shift 2;; + --log) LOG="$2"; shift 2;; + --ckpt-path) CKPT_PATH="$2"; shift 2;; + --run-name) RUN_NAME="$2"; shift 2;; + --container) CONTAINER="$2"; shift 2;; + --max-relaunch) MAX_RELAUNCH="$2"; shift 2;; + --min-free-gib) MIN_FREE_GIB="$2"; shift 2;; + --stall-s) STALL_S="$2"; shift 2;; + --poll-s) POLL_S="$2"; shift 2;; + *) echo "Unknown arg: $1"; exit 1;; + esac +done + +[[ -n "$SUBMIT_SCRIPT" && -f "$SUBMIT_SCRIPT" ]] || { echo "FATAL: --submit-script required and must exist"; exit 1; } +[[ -n "$LOG" ]] || { echo "FATAL: --log required"; exit 1; } + +SUP_LOG="${LOG%.log}.supervisor.log" +sup() { echo "[$(date '+%F %T')] [supervisor] $*" | tee -a "$SUP_LOG"; } + +# Is the job in the queue right now (single read)? +job_in_queue() { [[ -n "$(squeue -h -j "$1" -o '%T' 2>/dev/null | head -1)" ]]; } +job_state() { squeue -h -j "$1" -o '%T' 2>/dev/null | head -1; } + +# Is the job still active? squeue/the SLURM control plane can transiently return +# empty during an NFS/controller blip even though the job is alive (this once +# killed all supervisors at once: empty squeue -> sacct said RUNNING -> a bogus +# "relaunch"). So a SINGLE empty read is not trusted: re-check a few times before +# believing the job is really gone. +job_active() { + job_in_queue "$1" && return 0 + local k + for k in 1 2 3; do + sleep 10 + job_in_queue "$1" && return 0 + done + return 1 +} + +# Terminal State + ExitCode from accounting once the job has left the queue. +job_final() { sacct -j "$1" -X -n -o State,ExitCode 2>/dev/null | head -1 | tr -s ' '; } + +# sacct/SLURM states that mean the job is STILL ALIVE (not terminal). If we see +# one of these after the monitor loop exits, squeue lied (transient) — resume +# monitoring instead of relaunching (which could spawn a DUPLICATE job). +is_active_state() { + case "$1" in + RUNNING|PENDING|CONFIGURING|COMPLETING|REQUEUED|RESIZING|SUSPENDED|REQUEUE_HOLD|REQUEUE_FED|SIGNALING|STAGE_OUT) return 0;; + *) return 1;; + esac +} + +# Any trainer process alive on ANY node of the allocation? (cross-node pgrep via +# overlap srun into each node's container). [g]enerative self-match guard avoids +# pgrep matching its own command line. +trainer_alive() { + local jid="$1" n + n=$(timeout 70 srun --jobid="$jid" --overlap --ntasks-per-node=1 bash -c \ + "docker exec $CONTAINER bash -lc 'set -f; pgrep -f [g]enerative_recommenders | wc -l' 2>/dev/null" 2>/dev/null \ + | awk '{s+=$1} END{print s+0}') + [[ "${n:-0}" -gt 0 ]] +} + +# Free GiB on the ckpt volume (NFS is mounted on this head node, so df locally). +disk_free_gib() { + df -BG --output=avail "$CKPT_PATH" 2>/dev/null | tail -1 | tr -dc '0-9' +} + +disk_guard() { + [[ -z "$CKPT_PATH" ]] && return 0 + local free; free=$(disk_free_gib); free=${free:-0} + sup "disk guard: ${free} GiB free on $CKPT_PATH (min ${MIN_FREE_GIB})" + if (( free < MIN_FREE_GIB )); then + sup "FATAL: insufficient free space (${free} < ${MIN_FREE_GIB} GiB). Aborting." + return 1 + fi + return 0 +} + +# Resubmit the run; resumes from latest checkpoint. APPEND_LOG=1 preserves the +# metrics log. Echoes the new jobid. +resubmit() { + local out newjid + out=$(APPEND_LOG=1 bash "$SUBMIT_SCRIPT" 2>&1) + newjid=$(echo "$out" | grep -oE 'Submitted batch job [0-9]+' | grep -oE '[0-9]+' | head -1) + echo "$out" | sed 's/^/ /' >> "$SUP_LOG" + echo "$newjid" +} + +# Submit with retries+backoff. A transient NFS / control-plane error (e.g. +# "sbatch: error: ... I/O error writing script/environment to file") must NOT +# kill the supervisor — it leaves runs unsupervised / unlaunched. Echoes a jobid +# on success, or empty after all retries. +submit_retry() { + local cand sub_try + for sub_try in $(seq 1 12); do + cand=$(resubmit) + if [[ "$cand" =~ ^[0-9]+$ ]]; then echo "$cand"; return 0; fi + sup "submit attempt $sub_try/12 failed (transient sbatch/NFS error) — backing off." + sleep $(( sub_try < 5 ? 30 : 120 )) + done + return 1 +} + +sup "=== streaming e2e supervisor start ===" +sup "run=$RUN_NAME submit=$SUBMIT_SCRIPT log=$LOG ckpt=$CKPT_PATH" +sup "max_relaunch=$MAX_RELAUNCH min_free_gib=$MIN_FREE_GIB stall_s=$STALL_S poll_s=$POLL_S" + +attempt=0 +if [[ -z "$JOBID" ]]; then + if ! disk_guard; then exit 3; fi + sup "no --jobid given; submitting a fresh job" + JOBID=$(submit_retry) + [[ "$JOBID" =~ ^[0-9]+$ ]] || { sup "FATAL: could not submit after 12 retries — aborting."; exit 1; } +fi +attempt=1 +sup "supervising jobid=$JOBID (attempt $attempt/$MAX_RELAUNCH)" + +while (( attempt <= MAX_RELAUNCH )); do + # --- wait for the job to be schedulable / running --- + wait_pend=0 + while job_active "$JOBID" && [[ "$(job_state "$JOBID")" != "RUNNING" ]]; do + (( wait_pend % 10 == 0 )) && sup "job $JOBID state=$(job_state "$JOBID") — waiting to run…" + sleep "$POLL_S"; wait_pend=$((wait_pend + 1)) + done + [[ "$(job_state "$JOBID")" == "RUNNING" ]] && sup "job $JOBID RUNNING on $(squeue -h -j "$JOBID" -o '%N' 2>/dev/null | head -1)" + + # --- monitor loop --- + last_size=0; stall_accum=0; hb=0; self_cancelled=0 + while job_active "$JOBID"; do + st=$(job_state "$JOBID") + if [[ "$st" == "RUNNING" ]]; then + cur_size=$(stat -c %s "$LOG" 2>/dev/null || echo 0) + if [[ "$cur_size" == "$last_size" ]]; then + # frozen log: only count as a stall if no trainer proc is alive + # (a long eval / blocking save keeps the process up -> not a stall) + hb=$((hb + 1)) + if (( hb % 4 == 0 )); then + if trainer_alive "$JOBID"; then + stall_accum=0 + else + stall_accum=$((stall_accum + POLL_S * 4)) + sup "log frozen + no trainer alive (${stall_accum}s/${STALL_S}s)" + if (( stall_accum >= STALL_S )); then + sup "STALL: hung run — scancel $JOBID and relaunch." + self_cancelled=1 + scancel "$JOBID" 2>/dev/null || true + sleep 20 + break + fi + fi + fi + else + stall_accum=0; last_size=$cur_size + fi + fi + sleep "$POLL_S" + done + + # --- job has left the queue (or we scancel'd it): decide --- + sleep 5 + final=$(job_final "$JOBID") + state=$(echo "$final" | awk '{print $1}') + code=$(echo "$final" | awk '{print $2}') + sup "job $JOBID ended: state='${state:-?}' exit='${code:-?}'" + + # The monitor loop only exits when squeue has been empty across several + # confirming reads. If accounting STILL reports an active state, the job is + # actually alive (squeue/control-plane blip) — resume monitoring rather than + # relaunching, which would create a duplicate job. + if is_active_state "$state"; then + sup "sacct reports still-active state '$state' — transient squeue blip; resuming monitoring (NOT relaunching)." + sleep "$POLL_S" + continue + fi + + case "$state" in + COMPLETED) + if [[ "$code" == "0:0" ]]; then + sup "RUN COMPLETED CLEANLY on attempt $attempt." + sup "=== supervisor done (success) ===" + exit 0 + fi + sup "COMPLETED but nonzero exit ($code) — relaunching." + ;; + CANCELLED*) + if (( self_cancelled )); then + sup "job CANCELLED by our own stall recovery — relaunching from latest checkpoint." + else + sup "job CANCELLED (user/admin intent) — NOT resubmitting. Stopping supervisor." + sup "=== supervisor done (cancelled) ===" + exit 0 + fi + ;; + FAILED|NODE_FAIL|TIMEOUT|OUT_OF_MEMORY|BOOT_FAIL|PREEMPTED|"") + sup "failure state '${state:-unknown}' — will relaunch from latest checkpoint." + ;; + *) + sup "unrecognized terminal state '${state}' — relaunching to be safe." + ;; + esac + + if (( attempt >= MAX_RELAUNCH )); then break; fi + if ! disk_guard; then exit 3; fi + sleep $(( attempt < 5 ? 20 : 60 )) # small backoff + # Resubmit with retries. A transient NFS / control-plane error (e.g. + # "sbatch: error: Batch job submission failed: I/O error writing + # script/environment to file") must NOT kill the supervisor — that once + # left a live run permanently unsupervised. Retry with backoff first. + JOBID=$(submit_retry) + if ! [[ "$JOBID" =~ ^[0-9]+$ ]]; then + sup "FATAL: resubmit failed after 12 retries — aborting."; exit 1 + fi + attempt=$((attempt + 1)) + sup "relaunched as jobid=$JOBID (attempt $attempt/$MAX_RELAUNCH)" +done + +sup "FATAL: exhausted MAX_RELAUNCH=$MAX_RELAUNCH without completion." +sup "=== supervisor done (failure) ===" +exit 1 diff --git a/recommendation_v4/scripts/run_streaming_e2e_local.sh b/recommendation_v4/scripts/run_streaming_e2e_local.sh new file mode 100755 index 000000000..85a817c8c --- /dev/null +++ b/recommendation_v4/scripts/run_streaming_e2e_local.sh @@ -0,0 +1,200 @@ +#!/bin/bash +# ============================================================================= +# run_streaming_e2e_local.sh — self-healing supervisor for a SINGLE-HOST +# (NON-SLURM) yambda-5b streaming train+eval run. Local analog of +# scripts/run_streaming_e2e.sh (the SLURM/sbatch supervisor). +# +# WHAT IT SUPERVISES +# The "job" is one foreground run of --submit-script (default +# scripts/launch_e2e_local.sh), which `docker exec`s the trainer in the +# container. The supervisor runs that submit-script in the BACKGROUND so: +# * its host PID == liveness (kill -0 / hang watchdog), and +# * `wait $PID` == the trainer's EXIT CODE (success vs. failure). +# On crash / nonzero-exit / hang it RELAUNCHES the same submit-script (same +# $CKPT_PATH/$LOG → resumes from the latest checkpoint), bounded by +# --max-relaunch. This is the SLURM supervisor's sacct/squeue/scancel control +# plane re-expressed with a local process + `docker exec` lifecycle. +# +# WHAT IT DETECTS (poll every --poll-s) +# * submit-script process exits -> read its exit code: +# 0 => run finished cleanly (success) +# != 0 => crash/OOM/die_at_step(42)/etc. => relaunch from latest ckpt +# * hang watchdog: process alive but $LOG frozen >= --stall-s AND no trainer +# process alive in the container (pgrep via docker exec) => kill + pkill in +# container + relaunch. A long eval / blocking ckpt save keeps the trainer +# process up, so it is NOT counted as a stall. +# * disk guard before each (re)launch: require --min-free-gib on the ckpt vol. +# +# USAGE +# nohup bash scripts/run_streaming_e2e_local.sh \ +# --submit-script scripts/launch_e2e_local.sh \ +# --log /home/chcai/yambda_5b_e2e//.log \ +# --ckpt-path /home/chcai/yambda_5b_e2e//ckpts \ +# --run-name \ +# > /home/chcai/yambda_5b_e2e//.supervisor.console.log 2>&1 & +# +# Per-run hyperparameters live in the --submit-script's env defaults (or are +# exported before invoking this supervisor), not here. +# +# EXIT CODES +# 0 run completed cleanly +# 1 exhausted --max-relaunch without completion (or launch failed) +# 3 disk guard tripped +# ============================================================================= +set -uo pipefail + +SUBMIT_SCRIPT="scripts/launch_e2e_local.sh" +LOG="" +CKPT_PATH="" +RUN_NAME="yambda_5b_e2e_local" +CONTAINER=${CONTAINER:-yambda_local} +DOCKER=${DOCKER:-sudo docker} +MAX_RELAUNCH=50 +MIN_FREE_GIB=700 # one full DMP ckpt (~600 GB) + headroom for the atomic + # .tmp written beside the retained one during a save. +STALL_S=2400 # 40 min frozen-log + no-trainer-proc => hung. +POLL_S=30 + +while [[ $# -gt 0 ]]; do + case $1 in + --submit-script) SUBMIT_SCRIPT="$2"; shift 2;; + --log) LOG="$2"; shift 2;; + --ckpt-path) CKPT_PATH="$2"; shift 2;; + --run-name) RUN_NAME="$2"; shift 2;; + --container) CONTAINER="$2"; shift 2;; + --docker) DOCKER="$2"; shift 2;; + --max-relaunch) MAX_RELAUNCH="$2"; shift 2;; + --min-free-gib) MIN_FREE_GIB="$2"; shift 2;; + --stall-s) STALL_S="$2"; shift 2;; + --poll-s) POLL_S="$2"; shift 2;; + *) echo "Unknown arg: $1"; exit 1;; + esac +done + +[[ -n "$SUBMIT_SCRIPT" && -f "$SUBMIT_SCRIPT" ]] || { echo "FATAL: --submit-script required and must exist ($SUBMIT_SCRIPT)"; exit 1; } +[[ -n "$LOG" ]] || { echo "FATAL: --log required"; exit 1; } + +SUP_LOG="${LOG%.log}.supervisor.log" +# Create the log + ckpt dirs up front so the disk guard's df has a real path to +# stat (df on a nonexistent dir returns 0 avail -> false "disk full" abort). +mkdir -p "$(dirname "$SUP_LOG")" +[[ -n "$CKPT_PATH" ]] && mkdir -p "$CKPT_PATH" +sup() { echo "[$(date '+%F %T')] [supervisor] $*" | tee -a "$SUP_LOG"; } + +# Any trainer process alive in the container? [g]enerative self-match guard +# avoids pgrep matching its own command line. +trainer_alive() { + local n + n=$($DOCKER exec "$CONTAINER" bash -lc 'set -f; pgrep -f "[g]enerative_recommenders" | wc -l' 2>/dev/null | tr -dc '0-9') + [[ "${n:-0}" -gt 0 ]] +} + +# Hard-kill any trainer processes left in the container AND wait for GPU HBM to +# actually drain before returning. A rank stuck in a HIP/RCCL collective sits in +# uninterruptible D-state and keeps its multi-hundred-GB embedding shard resident +# for many seconds after SIGKILL; relaunching before that frees makes the next +# attempt OOM on dirty GPUs (an OOM-crash -> dirty-GPU -> OOM cascade). So kill, +# then poll rocm-smi until every GPU is <5 GB (or give up after ~120s). +cleanup_container() { + $DOCKER exec "$CONTAINER" bash -lc \ + 'pkill -9 -f generative_recommenders 2>/dev/null; pkill -9 -f spawn_main 2>/dev/null; pkill -9 -f resource_tracker 2>/dev/null; true' \ + 2>/dev/null || true + local k busy + for k in $(seq 1 24); do # up to ~120s + busy=$($DOCKER exec "$CONTAINER" bash -lc \ + "rocm-smi --showmeminfo vram 2>/dev/null | awk '/Used/{if (\$NF+0 > 5e9) c++} END{print c+0}'" \ + 2>/dev/null | tr -dc '0-9') + busy=${busy:-0} + [[ "$busy" == "0" ]] && return 0 + sup "waiting for GPU HBM to drain ($busy GPU(s) still >5GB)…" + $DOCKER exec "$CONTAINER" bash -lc 'pkill -9 -f spawn_main 2>/dev/null; true' 2>/dev/null || true + sleep 5 + done + sup "WARNING: GPUs still show residual HBM after 120s — launching anyway." + return 0 +} + +disk_free_gib() { df -BG --output=avail "$CKPT_PATH" 2>/dev/null | tail -1 | tr -dc '0-9'; } + +disk_guard() { + [[ -z "$CKPT_PATH" ]] && return 0 + local free; free=$(disk_free_gib); free=${free:-0} + sup "disk guard: ${free} GiB free on $CKPT_PATH (min ${MIN_FREE_GIB})" + if (( free < MIN_FREE_GIB )); then + sup "FATAL: insufficient free space (${free} < ${MIN_FREE_GIB} GiB). Aborting." + return 1 + fi + return 0 +} + +# Run the submit-script in the FOREGROUND (its exit status == the trainer's). +# APPEND_LOG=1 preserves the metrics log across relaunches. This is invoked as +# `launch & PID=$!` from the main loop so the backgrounded copy is a DIRECT child +# of this shell — otherwise `wait $PID` can't reap it and always returns 127, +# making every clean completion look like a failure (infinite relaunch loop). +launch() { + APPEND_LOG=1 CONTAINER="$CONTAINER" DOCKER="$DOCKER" \ + RUN_NAME="$RUN_NAME" LOG="$LOG" CKPT_PATH="$CKPT_PATH" \ + bash "$SUBMIT_SCRIPT" >>"$SUP_LOG" 2>&1 +} + +sup "=== streaming e2e LOCAL supervisor start ===" +sup "run=$RUN_NAME submit=$SUBMIT_SCRIPT log=$LOG ckpt=$CKPT_PATH container=$CONTAINER" +sup "max_relaunch=$MAX_RELAUNCH min_free_gib=$MIN_FREE_GIB stall_s=$STALL_S poll_s=$POLL_S" + +attempt=1 +while (( attempt <= MAX_RELAUNCH )); do + if ! disk_guard; then exit 3; fi + sup "launching attempt $attempt/$MAX_RELAUNCH" + cleanup_container # ensure no stragglers from a prior attempt + launch & PID=$! # direct child => wait $PID reaps the real rc + sup "submit-script running as host pid=$PID" + + # --- monitor loop --- + last_size=0; stall_accum=0; hb=0; hung=0 + while kill -0 "$PID" 2>/dev/null; do + cur_size=$(stat -c %s "$LOG" 2>/dev/null || echo 0) + if [[ "$cur_size" == "$last_size" ]]; then + hb=$((hb + 1)) + # Re-check liveness only every 4 polls (cheap docker exec amortized). + if (( hb % 4 == 0 )); then + if trainer_alive; then + stall_accum=0 + else + stall_accum=$((stall_accum + POLL_S * 4)) + sup "log frozen + no trainer alive (${stall_accum}s/${STALL_S}s)" + if (( stall_accum >= STALL_S )); then + sup "STALL: hung run — killing pid=$PID + container trainer procs, will relaunch." + hung=1 + kill -9 "$PID" 2>/dev/null || true + cleanup_container + break + fi + fi + fi + else + stall_accum=0; last_size=$cur_size + fi + sleep "$POLL_S" + done + + # --- the submit-script has exited (or we killed it): decide --- + wait "$PID" 2>/dev/null; rc=$? + if (( hung )); then + sup "attempt $attempt ended via STALL recovery (rc=$rc) — relaunching from latest checkpoint." + elif (( rc == 0 )); then + sup "RUN COMPLETED CLEANLY on attempt $attempt." + sup "=== supervisor done (success) ===" + exit 0 + else + sup "attempt $attempt exited rc=$rc (crash/OOM/die_at_step) — relaunching from latest checkpoint." + fi + + if (( attempt >= MAX_RELAUNCH )); then break; fi + sleep $(( attempt < 5 ? 20 : 60 )) # small backoff + attempt=$((attempt + 1)) +done + +sup "FATAL: exhausted MAX_RELAUNCH=$MAX_RELAUNCH without completion." +sup "=== supervisor done (failure) ===" +exit 1 diff --git a/recommendation_v4/scripts/stitch_traces.py b/recommendation_v4/scripts/stitch_traces.py new file mode 100644 index 000000000..54d7963d6 --- /dev/null +++ b/recommendation_v4/scripts/stitch_traces.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +"""Stitch per-rank Chrome traces from a dlrm_v3 run into one merged file. + +When ``Profiler`` runs on multiple ranks, each rank writes its own file: + + /trace_step{step}_rank{rank}.json + +Each per-rank trace uses overlapping ``pid`` namespaces (CPU pid = OS pid; +GPU streams pid = 0..N), so concatenating the raw event lists would collapse +multiple ranks onto the same Perfetto track. This script: + +* Identifies each pid as ``CPU`` / ``GPU`` / ``Spans`` (and other torch.profiler + string-pid tracks) using the per-rank ``process_labels`` metadata events. +* Always drops the ``Spans`` track (low-signal in this codebase, large in + visual clutter). +* Optionally filters to just ``cpu`` or ``gpu`` events via ``--include``. +* Sorts the surviving tracks into contiguous Perfetto sections: + **all CPU tracks (rank 0..N) first, then all GPU tracks (rank 0..N, stream + 0..K)**. +* Remaps every event's ``pid`` and flow ``id`` so cross-rank events never + collide on the same track or flow arrow. + +Because torch.profiler emits ``baseTimeNanoseconds`` from the same node clock, +timestamps line up directly across ranks — no time-shift needed for single-node +runs (multi-node would need clock-skew correction, not implemented here). + +Examples +-------- +Stitch step 52, default (CPU + GPU, drop Spans), gzip output:: + + python scripts/stitch_traces.py --step 52 --gzip + +GPU-only view (skip CPU thread tree entirely — useful for kernel-level analysis):: + + python scripts/stitch_traces.py --step 52 --include gpu --gzip + +CPU-only view (host-side ops, profiler annotations, comm scheduling):: + + python scripts/stitch_traces.py --step 52 --include cpu --gzip +""" +from __future__ import annotations + +import argparse +import gzip +import json +import re +import sys +from collections import defaultdict +from pathlib import Path + +# trace_step52_rank3.json or trace_3_rank0.json (legacy filename) +_RANK_RE = re.compile(r"trace_(?:step)?(\d+)_rank(\d+)\.json$") +_KEY_RE = re.compile(r"trace_(.+?)_rank\d+\.json$") + +# Per-rank pid offset. Picked large enough that no real OS pid collides +# (Linux pids fit in 22 bits; 1e6 per rank gives ~10 ranks of headroom). +_PID_STRIDE = 1_000_000 + +# Per-rank flow-id offset. torch.profiler flow ids are int32/int64 — pack rank +# into the high bits so cross-rank flows can never link by accident. +_FLOW_ID_STRIDE = 1 << 40 + +# Sort-index sections in Perfetto. Lower = appears higher in the timeline UI. +# Each section reserves a wide range so within-section ordering (rank, stream) +# fits comfortably without overlapping the next section. +_SORT_BASE = { + "cpu": 0, + "gpu": 1_000_000, + "other": 10_000_000, # Traces / "" misc string-pid tracks +} + +# `Spans` carries no useful content in our workloads (one X event per trace) +# and clutters the timeline — always dropped. +_ALWAYS_DROP_PIDS_STR = {"Spans"} + + +def _classify_pid(pid_to_label: dict, pid_to_name: dict) -> dict: + """Map original pid -> ('cpu'|'gpu'|'spans'|'other', stream_idx_or_0). + + Classification order, first match wins: + 1. pid (as a string) is in the always-drop set -> 'spans' + 2. process_name is in the always-drop set -> 'spans' + 3. process_labels == 'CPU' -> 'cpu' + 4. process_labels starts with 'GPU ' -> 'gpu', stream id + 5. anything else (including unlabeled pids) -> 'other' + """ + all_pids = set(pid_to_label) | set(pid_to_name) + out: dict = {} + for pid in all_pids: + label = pid_to_label.get(pid, "") + name = pid_to_name.get(pid, "") + if isinstance(pid, str) and pid in _ALWAYS_DROP_PIDS_STR: + out[pid] = ("spans", 0) + continue + if name in _ALWAYS_DROP_PIDS_STR: + out[pid] = ("spans", 0) + continue + if label == "CPU": + out[pid] = ("cpu", 0) + elif label.startswith("GPU"): + try: + stream_idx = int(label.split()[1]) + except (IndexError, ValueError): + stream_idx = 0 + out[pid] = ("gpu", stream_idx) + else: + out[pid] = ("other", 0) + return out + + +def _scan_pid_metadata(events: list[dict]) -> tuple[dict, dict]: + """First pass: collect per-pid label and name from ``ph='M'`` events.""" + label: dict = {} + name: dict = {} + for e in events: + if e.get("ph") != "M": + continue + pid = e.get("pid") + if pid is None: + continue + if e.get("name") == "process_labels": + label[pid] = e.get("args", {}).get("labels", "") + elif e.get("name") == "process_name": + name[pid] = e.get("args", {}).get("name", "") + return label, name + + +def _new_sort_index(kind: str, rank: int, stream_idx: int) -> int: + """Compute Perfetto sort_index so tracks group as: CPU(rank0..N), GPU(rank0..N, stream0..K), other.""" + base = _SORT_BASE.get(kind, _SORT_BASE["other"]) + return base + rank * 100 + stream_idx + + +def _new_pid(orig_pid, rank: int) -> object: + """Remap a single pid into a per-rank namespace, preserving int vs str.""" + if isinstance(orig_pid, int): + return orig_pid + rank * _PID_STRIDE + if isinstance(orig_pid, str): + try: + return int(orig_pid) + rank * _PID_STRIDE + except ValueError: + return f"rank{rank}_{orig_pid}" if orig_pid else f"rank{rank}_misc" + return orig_pid + + +def _process_one_rank( + events: list[dict], + rank: int, + include: set[str], +) -> list[dict]: + """Filter + remap one rank's events. ``include`` is a subset of {'cpu','gpu','other'}.""" + label, name = _scan_pid_metadata(events) + classify = _classify_pid(label, name) + + out: list[dict] = [] + for e in events: + pid = e.get("pid") + if pid is None: + out.append(e) + continue + # Always-drop check on the raw pid value first - Spans events in our + # workloads have NO process_name/process_labels metadata, so the + # classifier table doesn't list them. Catch them here directly. + if isinstance(pid, str) and pid in _ALWAYS_DROP_PIDS_STR: + continue + kind, stream_idx = classify.get(pid, ("other", 0)) + if kind == "spans": # always dropped + continue + if kind not in include: # filtered by --include + continue + + # Remap pid + flow id (per-rank namespace). + e["pid"] = _new_pid(pid, rank) + if "id" in e and e.get("ph") in ("s", "t", "f"): + try: + e["id"] = int(e["id"]) + rank * _FLOW_ID_STRIDE + except (TypeError, ValueError): + pass + + # Rewrite metadata: section-aware sort_index + rank-prefixed name. + if e.get("ph") == "M": + args = e.setdefault("args", {}) + if e.get("name") == "process_sort_index": + args["sort_index"] = _new_sort_index(kind, rank, stream_idx) + elif e.get("name") == "process_name": + orig = args.get("name", "python") + args["name"] = f"[Rank {rank}] {orig}" + + out.append(e) + + return out + + +def _group_by_step(trace_dir: Path) -> dict[str, dict[int, Path]]: + """Map step-key (e.g. ``"step52"`` or ``"3"``) -> {rank: path}.""" + groups: dict[str, dict[int, Path]] = defaultdict(dict) + for p in sorted(trace_dir.glob("trace_*_rank*.json")): + m = _RANK_RE.search(p.name) + if not m: + continue + prefix_match = _KEY_RE.match(p.name) + key = prefix_match.group(1) if prefix_match else m.group(1) + groups[key][int(m.group(2))] = p + return dict(groups) + + +def stitch_one(rank_to_path: dict[int, Path], out_path: Path, *, + include: set[str], gzip_out: bool, verbose: bool) -> None: + """Merge one (step, rank->path) group into a single trace file.""" + merged_events: list[dict] = [] + base: dict | None = None + + for rank in sorted(rank_to_path): + path = rank_to_path[rank] + if verbose: + sz_mb = path.stat().st_size / (1 << 20) + print(f" rank {rank}: {path.name} ({sz_mb:.1f} MB)", file=sys.stderr) + with path.open() as f: + trace = json.load(f) + if base is None: + base = {k: v for k, v in trace.items() if k != "traceEvents"} + base["distributedInfo"] = { + **trace.get("distributedInfo", {}), + "stitched_ranks": sorted(rank_to_path), + "stitched_files": [p.name for p in rank_to_path.values()], + "stitched_include": sorted(include), + } + merged_events.extend( + _process_one_rank(trace.get("traceEvents", []), rank, include) + ) + + assert base is not None, "no input traces provided" + base["traceEvents"] = merged_events + + out_path.parent.mkdir(parents=True, exist_ok=True) + if gzip_out: + with gzip.open(out_path, "wt") as f: + json.dump(base, f) + else: + with out_path.open("w") as f: + json.dump(base, f) + if verbose: + sz_mb = out_path.stat().st_size / (1 << 20) + print( + f" -> {out_path} ({len(merged_events):,} events, {sz_mb:.1f} MB)", + file=sys.stderr, + ) + + +def main() -> int: + ap = argparse.ArgumentParser( + description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter, + ) + ap.add_argument("trace_dir", type=Path, + help="Directory containing trace_*_rank*.json files.") + ap.add_argument("--step", type=str, default=None, + help="Stitch only the given step key (e.g. '52' or 'step52'). " + "Default: stitch every step group found.") + ap.add_argument("--out", type=Path, default=None, + help="Output path. Only valid when --step selects exactly " + "one group. Default: /trace_.json[.gz] " + "(or trace__cpu/_gpu when --include filters).") + ap.add_argument("--include", choices=("cpu", "gpu", "both"), default="both", + help="Which sections to keep: cpu-only tracks, gpu-only " + "tracks, or both (default). 'Spans' is always dropped.") + ap.add_argument("--gzip", action="store_true", + help="Write gzip-compressed JSON (Perfetto auto-detects).") + ap.add_argument("-q", "--quiet", action="store_true") + args = ap.parse_args() + + if not args.trace_dir.is_dir(): + print(f"error: {args.trace_dir} is not a directory", file=sys.stderr) + return 2 + + if args.include == "both": + # 'other' covers torch.profiler string-pid tracks (Traces / misc) that + # carry low-volume but legitimate annotations. Dropped under cpu/gpu + # so each filtered view is clean. + include = {"cpu", "gpu", "other"} + else: + include = {args.include} + + groups = _group_by_step(args.trace_dir) + if not groups: + print(f"error: no trace_*_rank*.json files under {args.trace_dir}", + file=sys.stderr) + return 2 + + if args.step is not None: + wanted = args.step if args.step.startswith("step") else f"step{args.step}" + if wanted not in groups and args.step in groups: + wanted = args.step + if wanted not in groups: + print( + f"error: step {args.step!r} not found. " + f"Available: {sorted(groups)}", + file=sys.stderr, + ) + return 2 + groups = {wanted: groups[wanted]} + + if args.out is not None and len(groups) != 1: + print("error: --out requires --step to select exactly one group", + file=sys.stderr) + return 2 + + for key, rank_map in sorted(groups.items()): + if not args.quiet: + print( + f"stitching {key} ({len(rank_map)} ranks, include={args.include}):", + file=sys.stderr, + ) + if args.out is not None: + out = args.out + else: + ext = ".json.gz" if args.gzip else ".json" + # Default mode ("both") gets the bare filename; explicit cpu/gpu + # filters tag the output so they can coexist in one directory. + suffix = "" if args.include == "both" else f"_{args.include}" + out = args.trace_dir / f"trace_{key}{suffix}{ext}" + stitch_one(rank_map, out, include=include, + gzip_out=args.gzip, verbose=not args.quiet) + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/recommendation_v4/setup.py b/recommendation_v4/setup.py new file mode 100644 index 000000000..bdab528f4 --- /dev/null +++ b/recommendation_v4/setup.py @@ -0,0 +1,41 @@ +# Copyright (c) Meta Platforms, Inc. and affiliates. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# pyre-unsafe + +from setuptools import find_packages, setup + +setup( + name="generative_recommenders", + version="0.1.0", + description="Library for generative recommendation algorithms.", + packages=find_packages(exclude=["configs"]), + python_requires=">=3.10", + install_requires=[ + "torch>=2.6.0", + "fbgemm_gpu>=1.1.0", + "torchrec>=1.1.0", + "gin_config>=0.5.0", + "pandas>=2.2.0", + "tensorboard>=2.19.0", + "pybind11", + "click", + "pandas", + "matplotlib", + ], + long_description=open("README.md").read(), + long_description_content_type="text/markdown", + url="https://github.com/meta-recsys/generative-recommenders", + license="Apache-2.0", +) diff --git a/recommendation_v4/verify_dataset.sh b/recommendation_v4/verify_dataset.sh new file mode 100755 index 000000000..839ccb91f --- /dev/null +++ b/recommendation_v4/verify_dataset.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# MLPerf Training reference script: verify the preprocessed dataset. +# +# Checks the integrity of the preprocessed dataset under +# ${DLRM_DATA_PATH}/${PROCESSED_SUBDIR} +# against md5sums_yambda_5b_processed.txt (standard `md5sum -c` format). +# +# If the checksum file still contains placeholder hashes (TODO_GENERATE_HASH), +# the script falls back to an existence/layout check and warns that the +# canonical checksums have not been pinned yet. +# +# Usage: +# DLRM_DATA_PATH=/path/to/dlrm_data ./verify_dataset.sh +# +# Env: +# DLRM_DATA_PATH data root (required). +# PROCESSED_SUBDIR processed subdir under the data root (default: processed_5b). +set -euo pipefail + +: "${DLRM_DATA_PATH:?Set DLRM_DATA_PATH to the data root}" +PROCESSED_SUBDIR="${PROCESSED_SUBDIR:-processed_5b}" + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CHECKSUM_FILE="${REPO_ROOT}/md5sums_yambda_5b_processed.txt" +PROCESSED_DIR="${DLRM_DATA_PATH}/${PROCESSED_SUBDIR}" + +echo "[verify_dataset] processed dir: ${PROCESSED_DIR}" + +if [[ ! -d "${PROCESSED_DIR}" ]]; then + echo "[verify_dataset] ERROR: ${PROCESSED_DIR} does not exist. Run ./download_dataset.sh first." >&2 + exit 1 +fi + +EXPECTED_FILES=( + train_sessions.parquet + test_events.parquet + session_index.parquet + item_popularity.npy + split_meta.json +) + +# Detect whether the checksum file has real (32 hex char) hashes or placeholders. +if grep -qiE '^[0-9a-f]{32}[[:space:]]' "${CHECKSUM_FILE}"; then + echo "[verify_dataset] checking md5 checksums from ${CHECKSUM_FILE}" + (cd "${PROCESSED_DIR}" && md5sum -c "${CHECKSUM_FILE}") + echo "[verify_dataset] OK: all checksums match." +else + echo "[verify_dataset] WARNING: ${CHECKSUM_FILE} contains placeholder hashes;" >&2 + echo "[verify_dataset] falling back to existence/layout check only." >&2 + missing=0 + for f in "${EXPECTED_FILES[@]}"; do + if [[ -s "${PROCESSED_DIR}/${f}" ]]; then + echo " OK ${f}" + else + echo " MISS ${f}" >&2 + missing=1 + fi + done + if [[ "${missing}" -ne 0 ]]; then + echo "[verify_dataset] ERROR: one or more expected files are missing/empty." >&2 + exit 1 + fi + echo "[verify_dataset] layout OK (checksums NOT yet pinned -- see TODO in ${CHECKSUM_FILE})." +fi